2024-11-18 20:08:11,880 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 20:08:11,896 main DEBUG Took 0.013912 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 20:08:11,897 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 20:08:11,897 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 20:08:11,898 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 20:08:11,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,911 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 20:08:11,928 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,930 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,931 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,931 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,932 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,933 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,934 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,936 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,938 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,939 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,940 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,941 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,943 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,944 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:08:11,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,945 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 20:08:11,947 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:08:11,949 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 20:08:11,952 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 20:08:11,952 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 20:08:11,954 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 20:08:11,954 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 20:08:11,964 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 20:08:11,967 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 20:08:11,969 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 20:08:11,970 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 20:08:11,970 main DEBUG createAppenders(={Console}) 2024-11-18 20:08:11,971 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-18 20:08:11,972 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 20:08:11,972 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-18 20:08:11,973 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 20:08:11,973 main DEBUG OutputStream closed 2024-11-18 20:08:11,974 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 20:08:11,974 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 20:08:11,974 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-18 20:08:12,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 20:08:12,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 20:08:12,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 20:08:12,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 20:08:12,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 20:08:12,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 20:08:12,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 20:08:12,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 20:08:12,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 20:08:12,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 20:08:12,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 20:08:12,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 20:08:12,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 20:08:12,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 20:08:12,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 20:08:12,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 20:08:12,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 20:08:12,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 20:08:12,074 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 20:08:12,074 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-18 20:08:12,075 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 20:08:12,075 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-18T20:08:12,371 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c 2024-11-18 20:08:12,374 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 20:08:12,374 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T20:08:12,383 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-18T20:08:12,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T20:08:12,426 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295, deleteOnExit=true 2024-11-18T20:08:12,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T20:08:12,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/test.cache.data in system properties and HBase conf 2024-11-18T20:08:12,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T20:08:12,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/hadoop.log.dir in system properties and HBase conf 2024-11-18T20:08:12,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T20:08:12,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T20:08:12,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T20:08:12,533 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T20:08:12,643 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T20:08:12,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T20:08:12,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T20:08:12,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T20:08:12,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T20:08:12,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T20:08:12,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T20:08:12,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T20:08:12,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T20:08:12,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T20:08:12,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/nfs.dump.dir in system properties and HBase conf 2024-11-18T20:08:12,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/java.io.tmpdir in system properties and HBase conf 2024-11-18T20:08:12,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T20:08:12,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T20:08:12,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T20:08:13,687 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T20:08:13,764 INFO [Time-limited test {}] log.Log(170): Logging initialized @2761ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T20:08:13,839 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:08:13,902 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T20:08:13,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T20:08:13,926 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T20:08:13,927 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T20:08:13,940 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:08:13,943 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/hadoop.log.dir/,AVAILABLE} 2024-11-18T20:08:13,945 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T20:08:14,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/java.io.tmpdir/jetty-localhost-38091-hadoop-hdfs-3_4_1-tests_jar-_-any-8704561505640605973/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T20:08:14,147 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38091} 2024-11-18T20:08:14,147 INFO [Time-limited test {}] server.Server(415): Started @3145ms 2024-11-18T20:08:14,686 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:08:14,696 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T20:08:14,697 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T20:08:14,698 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T20:08:14,698 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T20:08:14,699 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/hadoop.log.dir/,AVAILABLE} 2024-11-18T20:08:14,700 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T20:08:14,812 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/java.io.tmpdir/jetty-localhost-36717-hadoop-hdfs-3_4_1-tests_jar-_-any-14931352725443434340/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T20:08:14,813 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36717} 2024-11-18T20:08:14,813 INFO [Time-limited test {}] server.Server(415): Started @3810ms 2024-11-18T20:08:14,872 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T20:08:15,759 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/data/data1/current/BP-891117525-172.17.0.2-1731960493244/current, will proceed with Du for space computation calculation, 2024-11-18T20:08:15,759 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/data/data2/current/BP-891117525-172.17.0.2-1731960493244/current, will proceed with Du for space computation calculation, 2024-11-18T20:08:15,790 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T20:08:15,835 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11a41fe9a59b4be2 with lease ID 0x53e3563d29724206: Processing first storage report for DS-277a3521-ef6e-4a76-a40b-f0f6decb2492 from datanode DatanodeRegistration(127.0.0.1:35355, datanodeUuid=1977cf0f-cc6f-41c3-863b-3bc2700219f3, infoPort=44309, infoSecurePort=0, ipcPort=46337, storageInfo=lv=-57;cid=testClusterID;nsid=1804037297;c=1731960493244) 2024-11-18T20:08:15,837 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11a41fe9a59b4be2 with lease ID 0x53e3563d29724206: from storage DS-277a3521-ef6e-4a76-a40b-f0f6decb2492 node DatanodeRegistration(127.0.0.1:35355, datanodeUuid=1977cf0f-cc6f-41c3-863b-3bc2700219f3, infoPort=44309, infoSecurePort=0, ipcPort=46337, storageInfo=lv=-57;cid=testClusterID;nsid=1804037297;c=1731960493244), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T20:08:15,838 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11a41fe9a59b4be2 with lease ID 0x53e3563d29724206: Processing first storage report for DS-f915af13-fc4d-4420-b0cc-e78b9f61380e from datanode DatanodeRegistration(127.0.0.1:35355, datanodeUuid=1977cf0f-cc6f-41c3-863b-3bc2700219f3, infoPort=44309, infoSecurePort=0, ipcPort=46337, storageInfo=lv=-57;cid=testClusterID;nsid=1804037297;c=1731960493244) 2024-11-18T20:08:15,838 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11a41fe9a59b4be2 with lease ID 0x53e3563d29724206: from storage DS-f915af13-fc4d-4420-b0cc-e78b9f61380e node DatanodeRegistration(127.0.0.1:35355, datanodeUuid=1977cf0f-cc6f-41c3-863b-3bc2700219f3, infoPort=44309, infoSecurePort=0, ipcPort=46337, storageInfo=lv=-57;cid=testClusterID;nsid=1804037297;c=1731960493244), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T20:08:15,870 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c 2024-11-18T20:08:15,939 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/zookeeper_0, clientPort=58690, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T20:08:15,949 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58690 2024-11-18T20:08:15,959 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:15,961 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:16,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741825_1001 (size=7) 2024-11-18T20:08:16,588 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c with version=8 2024-11-18T20:08:16,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/hbase-staging 2024-11-18T20:08:16,672 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T20:08:16,897 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d8f95bb18b58:0 server-side Connection retries=45 2024-11-18T20:08:16,906 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:08:16,906 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T20:08:16,911 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T20:08:16,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:08:16,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T20:08:17,041 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T20:08:17,100 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T20:08:17,111 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T20:08:17,116 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T20:08:17,148 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 117348 (auto-detected) 2024-11-18T20:08:17,149 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T20:08:17,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42935 2024-11-18T20:08:17,199 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42935 connecting to ZooKeeper ensemble=127.0.0.1:58690 2024-11-18T20:08:17,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429350x0, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T20:08:17,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42935-0x10150d991190000 connected 2024-11-18T20:08:17,384 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:17,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:17,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:08:17,407 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c, hbase.cluster.distributed=false 2024-11-18T20:08:17,442 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T20:08:17,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42935 2024-11-18T20:08:17,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42935 2024-11-18T20:08:17,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42935 2024-11-18T20:08:17,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42935 2024-11-18T20:08:17,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42935 2024-11-18T20:08:17,575 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d8f95bb18b58:0 server-side Connection retries=45 2024-11-18T20:08:17,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:08:17,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T20:08:17,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T20:08:17,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:08:17,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T20:08:17,580 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T20:08:17,582 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T20:08:17,583 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42447 2024-11-18T20:08:17,586 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42447 connecting to ZooKeeper ensemble=127.0.0.1:58690 2024-11-18T20:08:17,588 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:17,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:17,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424470x0, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T20:08:17,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424470x0, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:08:17,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42447-0x10150d991190001 connected 2024-11-18T20:08:17,621 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T20:08:17,632 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T20:08:17,635 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T20:08:17,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T20:08:17,642 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42447 2024-11-18T20:08:17,642 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42447 2024-11-18T20:08:17,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42447 2024-11-18T20:08:17,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42447 2024-11-18T20:08:17,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42447 2024-11-18T20:08:17,660 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d8f95bb18b58:42935 2024-11-18T20:08:17,661 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:17,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:08:17,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:08:17,675 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:17,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:17,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T20:08:17,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:17,708 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T20:08:17,711 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d8f95bb18b58,42935,1731960496743 from backup master directory 2024-11-18T20:08:17,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:08:17,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:17,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:08:17,724 WARN [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T20:08:17,724 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:17,726 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T20:08:17,727 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T20:08:17,792 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/hbase.id] with ID: 10a39af9-8ba6-46dd-ada3-b4b0ab3511f2 2024-11-18T20:08:17,793 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/.tmp/hbase.id 2024-11-18T20:08:17,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741826_1002 (size=42) 2024-11-18T20:08:17,806 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/.tmp/hbase.id]:[hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/hbase.id] 2024-11-18T20:08:17,854 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:08:17,860 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T20:08:17,882 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-18T20:08:17,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:17,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741827_1003 (size=196) 2024-11-18T20:08:17,940 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:08:17,944 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T20:08:17,965 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:17,971 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:08:18,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741828_1004 (size=1189) 2024-11-18T20:08:18,428 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store 2024-11-18T20:08:18,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741829_1005 (size=34) 2024-11-18T20:08:18,853 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T20:08:18,856 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:18,858 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T20:08:18,858 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:08:18,858 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:08:18,859 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T20:08:18,860 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:08:18,860 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:08:18,861 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731960498857Disabling compacts and flushes for region at 1731960498857Disabling writes for close at 1731960498860 (+3 ms)Writing region close event to WAL at 1731960498860Closed at 1731960498860 2024-11-18T20:08:18,863 WARN [master/d8f95bb18b58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/.initializing 2024-11-18T20:08:18,863 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/WALs/d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:18,873 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T20:08:18,887 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d8f95bb18b58%2C42935%2C1731960496743, suffix=, logDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/WALs/d8f95bb18b58,42935,1731960496743, archiveDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/oldWALs, maxLogs=10 2024-11-18T20:08:18,911 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/WALs/d8f95bb18b58,42935,1731960496743/d8f95bb18b58%2C42935%2C1731960496743.1731960498892, exclude list is [], retry=0 2024-11-18T20:08:18,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35355,DS-277a3521-ef6e-4a76-a40b-f0f6decb2492,DISK] 2024-11-18T20:08:18,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T20:08:18,963 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/WALs/d8f95bb18b58,42935,1731960496743/d8f95bb18b58%2C42935%2C1731960496743.1731960498892 2024-11-18T20:08:18,964 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44309:44309)] 2024-11-18T20:08:18,965 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:08:18,965 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:18,968 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:18,969 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T20:08:19,032 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:19,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:19,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T20:08:19,039 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:19,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:19,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T20:08:19,043 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:19,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:19,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T20:08:19,047 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:19,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:19,049 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,052 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,054 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,059 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,060 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,063 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T20:08:19,067 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:08:19,071 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:08:19,073 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59166846, jitterRate=-0.11834529042243958}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T20:08:19,079 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731960498981Initializing all the Stores at 1731960498983 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960498984 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960498984Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960498985 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960498985Cleaning up temporary data from old regions at 1731960499060 (+75 ms)Region opened successfully at 1731960499078 (+18 ms) 2024-11-18T20:08:19,080 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T20:08:19,112 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb15e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d8f95bb18b58/172.17.0.2:0 2024-11-18T20:08:19,143 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T20:08:19,154 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T20:08:19,154 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T20:08:19,157 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T20:08:19,159 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T20:08:19,164 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T20:08:19,164 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T20:08:19,192 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T20:08:19,201 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T20:08:19,239 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T20:08:19,242 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T20:08:19,244 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T20:08:19,252 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T20:08:19,255 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T20:08:19,258 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T20:08:19,269 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T20:08:19,271 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T20:08:19,281 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T20:08:19,298 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T20:08:19,306 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T20:08:19,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T20:08:19,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T20:08:19,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:19,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:19,322 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d8f95bb18b58,42935,1731960496743, sessionid=0x10150d991190000, setting cluster-up flag (Was=false) 2024-11-18T20:08:19,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:19,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:19,373 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T20:08:19,375 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:19,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:19,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:19,422 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T20:08:19,425 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:19,432 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T20:08:19,449 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(746): ClusterId : 10a39af9-8ba6-46dd-ada3-b4b0ab3511f2 2024-11-18T20:08:19,452 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T20:08:19,470 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T20:08:19,471 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T20:08:19,483 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T20:08:19,484 DEBUG [RS:0;d8f95bb18b58:42447 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c4fe2b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d8f95bb18b58/172.17.0.2:0 2024-11-18T20:08:19,502 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d8f95bb18b58:42447 2024-11-18T20:08:19,505 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T20:08:19,505 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T20:08:19,505 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T20:08:19,508 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(2659): reportForDuty to master=d8f95bb18b58,42935,1731960496743 with port=42447, startcode=1731960497538 2024-11-18T20:08:19,516 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T20:08:19,521 DEBUG [RS:0;d8f95bb18b58:42447 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T20:08:19,527 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T20:08:19,534 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T20:08:19,540 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d8f95bb18b58,42935,1731960496743 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T20:08:19,547 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d8f95bb18b58:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:08:19,548 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d8f95bb18b58:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:08:19,548 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d8f95bb18b58:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:08:19,548 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d8f95bb18b58:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:08:19,549 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d8f95bb18b58:0, corePoolSize=10, maxPoolSize=10 2024-11-18T20:08:19,549 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,549 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d8f95bb18b58:0, corePoolSize=2, maxPoolSize=2 2024-11-18T20:08:19,550 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:08:19,556 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731960529556 2024-11-18T20:08:19,556 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T20:08:19,558 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T20:08:19,559 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T20:08:19,560 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49641, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T20:08:19,562 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T20:08:19,563 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:19,563 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T20:08:19,563 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T20:08:19,563 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T20:08:19,563 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T20:08:19,564 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,567 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:19,568 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T20:08:19,569 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T20:08:19,570 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T20:08:19,573 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T20:08:19,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741831_1007 (size=1321) 2024-11-18T20:08:19,574 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T20:08:19,575 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d8f95bb18b58:0:becomeActiveMaster-HFileCleaner.large.0-1731960499575,5,FailOnTimeoutGroup] 2024-11-18T20:08:19,577 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d8f95bb18b58:0:becomeActiveMaster-HFileCleaner.small.0-1731960499576,5,FailOnTimeoutGroup] 2024-11-18T20:08:19,578 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,578 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T20:08:19,579 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,580 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,592 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T20:08:19,592 WARN [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T20:08:19,694 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(2659): reportForDuty to master=d8f95bb18b58,42935,1731960496743 with port=42447, startcode=1731960497538 2024-11-18T20:08:19,696 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:19,699 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(517): Registering regionserver=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:19,708 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:08:19,708 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40539 2024-11-18T20:08:19,708 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T20:08:19,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T20:08:19,728 DEBUG [RS:0;d8f95bb18b58:42447 {}] zookeeper.ZKUtil(111): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:19,728 WARN [RS:0;d8f95bb18b58:42447 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T20:08:19,729 INFO [RS:0;d8f95bb18b58:42447 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:08:19,729 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:19,731 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d8f95bb18b58,42447,1731960497538] 2024-11-18T20:08:19,752 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T20:08:19,768 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T20:08:19,773 INFO [RS:0;d8f95bb18b58:42447 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T20:08:19,773 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,774 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T20:08:19,779 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T20:08:19,780 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,781 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,781 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,781 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,781 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,781 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,781 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d8f95bb18b58:0, corePoolSize=2, maxPoolSize=2 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d8f95bb18b58:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:08:19,782 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d8f95bb18b58:0, corePoolSize=3, maxPoolSize=3 2024-11-18T20:08:19,783 DEBUG [RS:0;d8f95bb18b58:42447 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0, corePoolSize=3, maxPoolSize=3 2024-11-18T20:08:19,783 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,784 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,784 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,784 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,784 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,784 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42447,1731960497538-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T20:08:19,803 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T20:08:19,805 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42447,1731960497538-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,805 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,806 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.Replication(171): d8f95bb18b58,42447,1731960497538 started 2024-11-18T20:08:19,823 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:19,824 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1482): Serving as d8f95bb18b58,42447,1731960497538, RpcServer on d8f95bb18b58/172.17.0.2:42447, sessionid=0x10150d991190001 2024-11-18T20:08:19,824 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T20:08:19,825 DEBUG [RS:0;d8f95bb18b58:42447 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:19,825 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd8f95bb18b58,42447,1731960497538' 2024-11-18T20:08:19,825 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T20:08:19,826 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T20:08:19,827 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T20:08:19,827 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T20:08:19,827 DEBUG [RS:0;d8f95bb18b58:42447 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:19,827 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd8f95bb18b58,42447,1731960497538' 2024-11-18T20:08:19,827 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T20:08:19,828 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T20:08:19,829 DEBUG [RS:0;d8f95bb18b58:42447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T20:08:19,829 INFO [RS:0;d8f95bb18b58:42447 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T20:08:19,829 INFO [RS:0;d8f95bb18b58:42447 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T20:08:19,934 INFO [RS:0;d8f95bb18b58:42447 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T20:08:19,938 INFO [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d8f95bb18b58%2C42447%2C1731960497538, suffix=, logDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538, archiveDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/oldWALs, maxLogs=32 2024-11-18T20:08:19,955 DEBUG [RS:0;d8f95bb18b58:42447 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538/d8f95bb18b58%2C42447%2C1731960497538.1731960499941, exclude list is [], retry=0 2024-11-18T20:08:19,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35355,DS-277a3521-ef6e-4a76-a40b-f0f6decb2492,DISK] 2024-11-18T20:08:19,963 INFO [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538/d8f95bb18b58%2C42447%2C1731960497538.1731960499941 2024-11-18T20:08:19,964 DEBUG [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44309:44309)] 2024-11-18T20:08:19,976 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T20:08:19,977 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:08:19,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741833_1009 (size=32) 2024-11-18T20:08:20,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:20,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T20:08:20,396 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T20:08:20,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T20:08:20,401 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T20:08:20,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T20:08:20,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T20:08:20,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T20:08:20,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T20:08:20,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,412 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T20:08:20,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740 2024-11-18T20:08:20,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740 2024-11-18T20:08:20,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T20:08:20,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T20:08:20,420 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:08:20,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T20:08:20,428 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:08:20,429 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60778041, jitterRate=-0.0943366140127182}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:08:20,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731960500390Initializing all the Stores at 1731960500392 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960500393 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960500393Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960500393Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960500393Cleaning up temporary data from old regions at 1731960500419 (+26 ms)Region opened successfully at 1731960500432 (+13 ms) 2024-11-18T20:08:20,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T20:08:20,432 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T20:08:20,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T20:08:20,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T20:08:20,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T20:08:20,434 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T20:08:20,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731960500432Disabling compacts and flushes for region at 1731960500432Disabling writes for close at 1731960500432Writing region close event to WAL at 1731960500433 (+1 ms)Closed at 1731960500434 (+1 ms) 2024-11-18T20:08:20,437 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:08:20,438 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T20:08:20,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T20:08:20,452 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T20:08:20,455 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T20:08:20,608 DEBUG [d8f95bb18b58:42935 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T20:08:20,621 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:20,628 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d8f95bb18b58,42447,1731960497538, state=OPENING 2024-11-18T20:08:20,664 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T20:08:20,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:20,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:08:20,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:08:20,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:08:20,675 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T20:08:20,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:08:20,849 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T20:08:20,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T20:08:20,866 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T20:08:20,867 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:08:20,868 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T20:08:20,871 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d8f95bb18b58%2C42447%2C1731960497538.meta, suffix=.meta, logDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538, archiveDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/oldWALs, maxLogs=32 2024-11-18T20:08:20,887 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538/d8f95bb18b58%2C42447%2C1731960497538.meta.1731960500873.meta, exclude list is [], retry=0 2024-11-18T20:08:20,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35355,DS-277a3521-ef6e-4a76-a40b-f0f6decb2492,DISK] 2024-11-18T20:08:20,894 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/WALs/d8f95bb18b58,42447,1731960497538/d8f95bb18b58%2C42447%2C1731960497538.meta.1731960500873.meta 2024-11-18T20:08:20,895 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44309:44309)] 2024-11-18T20:08:20,895 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:08:20,897 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T20:08:20,899 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T20:08:20,904 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T20:08:20,909 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T20:08:20,909 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:20,910 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T20:08:20,910 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T20:08:20,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T20:08:20,915 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T20:08:20,915 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T20:08:20,918 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T20:08:20,918 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T20:08:20,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T20:08:20,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T20:08:20,924 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T20:08:20,924 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:20,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:08:20,925 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T20:08:20,927 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740 2024-11-18T20:08:20,930 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740 2024-11-18T20:08:20,933 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T20:08:20,933 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T20:08:20,934 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:08:20,937 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T20:08:20,939 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71526390, jitterRate=0.06582626700401306}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:08:20,940 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T20:08:20,942 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731960500910Writing region info on filesystem at 1731960500910Initializing all the Stores at 1731960500912 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960500913 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960500913Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960500913Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960500913Cleaning up temporary data from old regions at 1731960500933 (+20 ms)Running coprocessor post-open hooks at 1731960500940 (+7 ms)Region opened successfully at 1731960500942 (+2 ms) 2024-11-18T20:08:20,950 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731960500841 2024-11-18T20:08:20,963 DEBUG [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T20:08:20,963 INFO [RS_OPEN_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T20:08:20,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:20,967 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d8f95bb18b58,42447,1731960497538, state=OPEN 2024-11-18T20:08:21,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T20:08:21,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T20:08:21,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:08:21,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:08:21,051 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:21,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T20:08:21,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d8f95bb18b58,42447,1731960497538 in 376 msec 2024-11-18T20:08:21,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T20:08:21,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-11-18T20:08:21,070 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:08:21,070 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T20:08:21,090 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:21,092 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:21,111 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:21,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40037, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:21,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6700 sec 2024-11-18T20:08:21,138 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731960501138, completionTime=-1 2024-11-18T20:08:21,142 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T20:08:21,143 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T20:08:21,173 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T20:08:21,173 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731960561173 2024-11-18T20:08:21,173 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731960621173 2024-11-18T20:08:21,173 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-18T20:08:21,176 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42935,1731960496743-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,176 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42935,1731960496743-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,176 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42935,1731960496743-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,178 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d8f95bb18b58:42935, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,178 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,179 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,185 DEBUG [master/d8f95bb18b58:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T20:08:21,233 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.508sec 2024-11-18T20:08:21,234 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T20:08:21,236 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T20:08:21,238 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T20:08:21,239 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T20:08:21,239 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T20:08:21,241 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42935,1731960496743-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T20:08:21,241 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42935,1731960496743-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T20:08:21,266 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T20:08:21,267 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T20:08:21,267 INFO [master/d8f95bb18b58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d8f95bb18b58,42935,1731960496743-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:08:21,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4346ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:21,276 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T20:08:21,277 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T20:08:21,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:21,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:21,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e3743c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:21,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:21,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:21,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:21,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:21,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:21,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=441, ProcessCount=11, AvailableMemoryMB=1737 2024-11-18T20:08:21,380 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:08:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:21,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57db5d72 2024-11-18T20:08:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:08:21,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:08:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:08:21,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:08:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:21,416 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:08:21,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-18T20:08:21,419 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:21,422 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:08:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:08:21,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741835_1011 (size=959) 2024-11-18T20:08:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:08:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:08:21,836 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:08:21,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741836_1012 (size=53) 2024-11-18T20:08:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:08:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b989f3762007617cfc5051d63359b50d, disabling compactions & flushes 2024-11-18T20:08:22,247 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. after waiting 0 ms 2024-11-18T20:08:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,247 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b989f3762007617cfc5051d63359b50d: Waiting for close lock at 1731960502247Disabling compacts and flushes for region at 1731960502247Disabling writes for close at 1731960502247Writing region close event to WAL at 1731960502247Closed at 1731960502247 2024-11-18T20:08:22,250 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:08:22,255 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960502250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960502250"}]},"ts":"1731960502250"} 2024-11-18T20:08:22,260 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:08:22,262 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:08:22,265 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960502262"}]},"ts":"1731960502262"} 2024-11-18T20:08:22,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:08:22,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, ASSIGN}] 2024-11-18T20:08:22,274 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, ASSIGN 2024-11-18T20:08:22,276 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, ASSIGN; state=OFFLINE, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=false 2024-11-18T20:08:22,427 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b989f3762007617cfc5051d63359b50d, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:22,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, ASSIGN because future has completed 2024-11-18T20:08:22,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:08:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:08:22,595 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,595 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:08:22,596 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,596 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:22,596 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,596 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,599 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,602 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b989f3762007617cfc5051d63359b50d columnFamilyName A 2024-11-18T20:08:22,602 DEBUG [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:22,604 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] regionserver.HStore(327): Store=b989f3762007617cfc5051d63359b50d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:22,604 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,607 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b989f3762007617cfc5051d63359b50d columnFamilyName B 2024-11-18T20:08:22,607 DEBUG [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:22,608 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] regionserver.HStore(327): Store=b989f3762007617cfc5051d63359b50d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:22,608 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,611 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b989f3762007617cfc5051d63359b50d columnFamilyName C 2024-11-18T20:08:22,611 DEBUG [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:22,612 INFO [StoreOpener-b989f3762007617cfc5051d63359b50d-1 {}] regionserver.HStore(327): Store=b989f3762007617cfc5051d63359b50d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:22,613 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,614 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,615 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,618 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,618 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,619 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:08:22,622 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,626 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:08:22,627 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b989f3762007617cfc5051d63359b50d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60087625, jitterRate=-0.10462461411952972}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:08:22,627 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:22,628 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b989f3762007617cfc5051d63359b50d: Running coprocessor pre-open hook at 1731960502596Writing region info on filesystem at 1731960502596Initializing all the Stores at 1731960502598 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960502598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960502598Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960502598Cleaning up temporary data from old regions at 1731960502618 (+20 ms)Running coprocessor post-open hooks at 1731960502627 (+9 ms)Region opened successfully at 1731960502628 (+1 ms) 2024-11-18T20:08:22,630 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., pid=6, masterSystemTime=1731960502588 2024-11-18T20:08:22,634 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,634 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:22,635 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b989f3762007617cfc5051d63359b50d, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:22,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:08:22,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T20:08:22,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 in 208 msec 2024-11-18T20:08:22,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T20:08:22,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, ASSIGN in 375 msec 2024-11-18T20:08:22,654 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:08:22,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960502654"}]},"ts":"1731960502654"} 2024-11-18T20:08:22,658 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:08:22,660 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:08:22,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2520 sec 2024-11-18T20:08:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:08:23,576 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:23,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,586 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c6058d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,589 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e5dcf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,591 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:08:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10ab891d 2024-11-18T20:08:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:08:23,604 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:08:23,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,610 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14496fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,613 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,618 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4c9506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,622 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36644156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,626 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,637 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615aad25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,640 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b688ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,643 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344fa400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eac2906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a3e301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,653 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3b3fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,659 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6e851d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29434e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:23,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:23,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68732d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:23,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce716ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:23,674 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,674 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,676 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,677 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:23,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-18T20:08:23,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,694 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:23,695 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,695 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,710 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,720 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,733 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:23,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:08:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:23,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:23,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:23,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:23,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:23,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81da7a93e1714736b5df02123626d5cc is 50, key is test_row_0/A:col10/1731960503734/Put/seqid=0 2024-11-18T20:08:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741837_1013 (size=12001) 2024-11-18T20:08:23,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81da7a93e1714736b5df02123626d5cc 2024-11-18T20:08:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:23,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960563967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960563987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960563987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960563976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960563976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:24,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:24,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/02524f77fe1447bb910c3e3e8b5726e9 is 50, key is test_row_0/B:col10/1731960503734/Put/seqid=0 2024-11-18T20:08:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960564113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960564113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741838_1014 (size=12001) 2024-11-18T20:08:24,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/02524f77fe1447bb910c3e3e8b5726e9 2024-11-18T20:08:24,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960564128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960564130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960564131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/8bc08c165e414b71885c043c523adfa4 is 50, key is test_row_0/C:col10/1731960503734/Put/seqid=0 2024-11-18T20:08:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741839_1015 (size=12001) 2024-11-18T20:08:24,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:24,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/8bc08c165e414b71885c043c523adfa4 2024-11-18T20:08:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81da7a93e1714736b5df02123626d5cc as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81da7a93e1714736b5df02123626d5cc 2024-11-18T20:08:24,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81da7a93e1714736b5df02123626d5cc, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T20:08:24,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/02524f77fe1447bb910c3e3e8b5726e9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/02524f77fe1447bb910c3e3e8b5726e9 2024-11-18T20:08:24,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/02524f77fe1447bb910c3e3e8b5726e9, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T20:08:24,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/8bc08c165e414b71885c043c523adfa4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/8bc08c165e414b71885c043c523adfa4 2024-11-18T20:08:24,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/8bc08c165e414b71885c043c523adfa4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T20:08:24,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b989f3762007617cfc5051d63359b50d in 534ms, sequenceid=14, compaction requested=false 2024-11-18T20:08:24,323 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:08:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:24,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:24,337 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:24,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:08:24,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/0e8b756a49184ac18e358ac898dafe9c is 50, key is test_row_0/A:col10/1731960504331/Put/seqid=0 2024-11-18T20:08:24,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:24,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960564385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960564390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960564396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960564402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960564411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741840_1016 (size=16681) 2024-11-18T20:08:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960564513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960564534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960564535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960564534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960564535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:24,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:24,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:24,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960564733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960564742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960564743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960564753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:24,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/0e8b756a49184ac18e358ac898dafe9c 2024-11-18T20:08:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:24,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/56418585ff524065994e1e8741b7ec0e is 50, key is test_row_0/B:col10/1731960504331/Put/seqid=0 2024-11-18T20:08:24,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:24,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:24,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:24,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:24,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741841_1017 (size=12001) 2024-11-18T20:08:24,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/56418585ff524065994e1e8741b7ec0e 2024-11-18T20:08:24,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/d769af64f65040d1b8059f33cb988804 is 50, key is test_row_0/C:col10/1731960504331/Put/seqid=0 2024-11-18T20:08:24,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741842_1018 (size=12001) 2024-11-18T20:08:24,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/d769af64f65040d1b8059f33cb988804 2024-11-18T20:08:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/0e8b756a49184ac18e358ac898dafe9c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/0e8b756a49184ac18e358ac898dafe9c 2024-11-18T20:08:25,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/0e8b756a49184ac18e358ac898dafe9c, entries=250, sequenceid=39, filesize=16.3 K 2024-11-18T20:08:25,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/56418585ff524065994e1e8741b7ec0e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/56418585ff524065994e1e8741b7ec0e 2024-11-18T20:08:25,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:25,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960565052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/56418585ff524065994e1e8741b7ec0e, entries=150, sequenceid=39, filesize=11.7 K 2024-11-18T20:08:25,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/d769af64f65040d1b8059f33cb988804 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d769af64f65040d1b8059f33cb988804 2024-11-18T20:08:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960565064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960565064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960565075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d769af64f65040d1b8059f33cb988804, entries=150, sequenceid=39, filesize=11.7 K 2024-11-18T20:08:25,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b989f3762007617cfc5051d63359b50d in 745ms, sequenceid=39, compaction requested=false 2024-11-18T20:08:25,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:25,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:08:25,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/800948c99f234ad6a4d566a66dfb168b is 50, key is test_row_0/A:col10/1731960504391/Put/seqid=0 2024-11-18T20:08:25,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741843_1019 (size=14341) 2024-11-18T20:08:25,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/800948c99f234ad6a4d566a66dfb168b 2024-11-18T20:08:25,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/e18416a9205f46149dba5b84fc636001 is 50, key is test_row_0/B:col10/1731960504391/Put/seqid=0 2024-11-18T20:08:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741844_1020 (size=12001) 2024-11-18T20:08:25,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/e18416a9205f46149dba5b84fc636001 2024-11-18T20:08:25,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/fb597b3d7b7d4117ac71807bb5d9bbcd is 50, key is test_row_0/C:col10/1731960504391/Put/seqid=0 2024-11-18T20:08:25,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960565342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741845_1021 (size=12001) 2024-11-18T20:08:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/fb597b3d7b7d4117ac71807bb5d9bbcd 2024-11-18T20:08:25,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/800948c99f234ad6a4d566a66dfb168b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/800948c99f234ad6a4d566a66dfb168b 2024-11-18T20:08:25,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/800948c99f234ad6a4d566a66dfb168b, entries=200, sequenceid=51, filesize=14.0 K 2024-11-18T20:08:25,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/e18416a9205f46149dba5b84fc636001 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e18416a9205f46149dba5b84fc636001 2024-11-18T20:08:25,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e18416a9205f46149dba5b84fc636001, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:08:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/fb597b3d7b7d4117ac71807bb5d9bbcd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fb597b3d7b7d4117ac71807bb5d9bbcd 2024-11-18T20:08:25,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960565453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fb597b3d7b7d4117ac71807bb5d9bbcd, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:08:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b989f3762007617cfc5051d63359b50d in 375ms, sequenceid=51, compaction requested=true 2024-11-18T20:08:25,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:25,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:25,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:25,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:25,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:25,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:25,477 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:25,481 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:25,483 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:25,484 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:25,484 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,485 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/02524f77fe1447bb910c3e3e8b5726e9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/56418585ff524065994e1e8741b7ec0e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e18416a9205f46149dba5b84fc636001] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=35.2 K 2024-11-18T20:08:25,487 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:25,487 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:25,487 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 02524f77fe1447bb910c3e3e8b5726e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731960503734 2024-11-18T20:08:25,487 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,487 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81da7a93e1714736b5df02123626d5cc, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/0e8b756a49184ac18e358ac898dafe9c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/800948c99f234ad6a4d566a66dfb168b] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=42.0 K 2024-11-18T20:08:25,488 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 56418585ff524065994e1e8741b7ec0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960503938 2024-11-18T20:08:25,489 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81da7a93e1714736b5df02123626d5cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731960503734 2024-11-18T20:08:25,490 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e18416a9205f46149dba5b84fc636001, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960504384 2024-11-18T20:08:25,490 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e8b756a49184ac18e358ac898dafe9c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960503938 2024-11-18T20:08:25,491 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800948c99f234ad6a4d566a66dfb168b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960504384 2024-11-18T20:08:25,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:08:25,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:08:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/df6b41aa51a44725909f6fe7cdbf4d46 is 50, key is test_row_0/A:col10/1731960505329/Put/seqid=0 2024-11-18T20:08:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741846_1022 (size=12001) 2024-11-18T20:08:25,566 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:25,566 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#10 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:25,567 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29 is 50, key is test_row_0/B:col10/1731960504391/Put/seqid=0 2024-11-18T20:08:25,567 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ade3b8211b56448fa1b428b040ca7422 is 50, key is test_row_0/A:col10/1731960504391/Put/seqid=0 2024-11-18T20:08:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:25,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741847_1023 (size=12104) 2024-11-18T20:08:25,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741848_1024 (size=12104) 2024-11-18T20:08:25,617 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ade3b8211b56448fa1b428b040ca7422 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ade3b8211b56448fa1b428b040ca7422 2024-11-18T20:08:25,617 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29 2024-11-18T20:08:25,642 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into ade3b8211b56448fa1b428b040ca7422(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:25,642 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:25,644 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 3e4cc6ccfd5e410f8b79d7b76d2a7a29(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:25,644 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:25,646 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960505474; duration=0sec 2024-11-18T20:08:25,646 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:25,647 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:25,647 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:25,648 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960505462; duration=0sec 2024-11-18T20:08:25,648 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:25,648 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:25,652 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:25,652 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:25,652 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:25,653 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/8bc08c165e414b71885c043c523adfa4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d769af64f65040d1b8059f33cb988804, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fb597b3d7b7d4117ac71807bb5d9bbcd] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=35.2 K 2024-11-18T20:08:25,654 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc08c165e414b71885c043c523adfa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731960503734 2024-11-18T20:08:25,656 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d769af64f65040d1b8059f33cb988804, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960503938 2024-11-18T20:08:25,657 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting fb597b3d7b7d4117ac71807bb5d9bbcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960504384 2024-11-18T20:08:25,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960565628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960565630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960565632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960565657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960565663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,709 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:25,709 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/1f085217667c4e96b8347bb53ea22b05 is 50, key is test_row_0/C:col10/1731960504391/Put/seqid=0 2024-11-18T20:08:25,748 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T20:08:25,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741849_1025 (size=12104) 2024-11-18T20:08:25,779 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/1f085217667c4e96b8347bb53ea22b05 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1f085217667c4e96b8347bb53ea22b05 2024-11-18T20:08:25,793 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 1f085217667c4e96b8347bb53ea22b05(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:25,793 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:25,794 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960505474; duration=0sec 2024-11-18T20:08:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,794 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:25,794 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960565776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960565776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960565776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960565782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:25,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/df6b41aa51a44725909f6fe7cdbf4d46 2024-11-18T20:08:25,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/878027fe0e2e437994787efdacefeb2e is 50, key is test_row_0/B:col10/1731960505329/Put/seqid=0 2024-11-18T20:08:25,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960565993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960566002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741850_1026 (size=12001) 2024-11-18T20:08:26,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960566003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960566003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,016 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/878027fe0e2e437994787efdacefeb2e 2024-11-18T20:08:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960566013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/516d98c335fb468f96b9bf86349efb19 is 50, key is test_row_0/C:col10/1731960505329/Put/seqid=0 2024-11-18T20:08:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741851_1027 (size=12001) 2024-11-18T20:08:26,068 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/516d98c335fb468f96b9bf86349efb19 2024-11-18T20:08:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/df6b41aa51a44725909f6fe7cdbf4d46 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/df6b41aa51a44725909f6fe7cdbf4d46 2024-11-18T20:08:26,107 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/df6b41aa51a44725909f6fe7cdbf4d46, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:08:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/878027fe0e2e437994787efdacefeb2e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/878027fe0e2e437994787efdacefeb2e 2024-11-18T20:08:26,130 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/878027fe0e2e437994787efdacefeb2e, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:08:26,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/516d98c335fb468f96b9bf86349efb19 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/516d98c335fb468f96b9bf86349efb19 2024-11-18T20:08:26,151 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/516d98c335fb468f96b9bf86349efb19, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:08:26,154 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b989f3762007617cfc5051d63359b50d in 645ms, sequenceid=75, compaction requested=false 2024-11-18T20:08:26,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:26,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:26,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-18T20:08:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-18T20:08:26,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T20:08:26,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4600 sec 2024-11-18T20:08:26,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.4880 sec 2024-11-18T20:08:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:26,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:26,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:08:26,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e06ef3de4be24624b018f6e2cb473947 is 50, key is test_row_0/A:col10/1731960505597/Put/seqid=0 2024-11-18T20:08:26,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741852_1028 (size=11997) 2024-11-18T20:08:26,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e06ef3de4be24624b018f6e2cb473947 2024-11-18T20:08:26,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/82d223610c204183986a82d901edf919 is 50, key is test_row_0/B:col10/1731960505597/Put/seqid=0 2024-11-18T20:08:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:08:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960566418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960566419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741853_1029 (size=9657) 2024-11-18T20:08:26,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/82d223610c204183986a82d901edf919 2024-11-18T20:08:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960566425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960566425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/95adccf8e77643aab90ca613bb0ff395 is 50, key is test_row_0/C:col10/1731960505597/Put/seqid=0 2024-11-18T20:08:26,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741854_1030 (size=9657) 2024-11-18T20:08:26,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/95adccf8e77643aab90ca613bb0ff395 2024-11-18T20:08:26,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960566502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e06ef3de4be24624b018f6e2cb473947 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e06ef3de4be24624b018f6e2cb473947 2024-11-18T20:08:26,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e06ef3de4be24624b018f6e2cb473947, entries=150, sequenceid=91, filesize=11.7 K 2024-11-18T20:08:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960566533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960566532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/82d223610c204183986a82d901edf919 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/82d223610c204183986a82d901edf919 2024-11-18T20:08:26,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/82d223610c204183986a82d901edf919, entries=100, sequenceid=91, filesize=9.4 K 2024-11-18T20:08:26,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960566553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960566553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/95adccf8e77643aab90ca613bb0ff395 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/95adccf8e77643aab90ca613bb0ff395 2024-11-18T20:08:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/95adccf8e77643aab90ca613bb0ff395, entries=100, sequenceid=91, filesize=9.4 K 2024-11-18T20:08:26,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for b989f3762007617cfc5051d63359b50d in 259ms, sequenceid=91, compaction requested=true 2024-11-18T20:08:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:26,579 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:26,579 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:26,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:26,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:26,582 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:26,582 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:26,582 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:26,582 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:26,582 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:26,582 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:26,583 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ade3b8211b56448fa1b428b040ca7422, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/df6b41aa51a44725909f6fe7cdbf4d46, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e06ef3de4be24624b018f6e2cb473947] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=35.3 K 2024-11-18T20:08:26,583 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/878027fe0e2e437994787efdacefeb2e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/82d223610c204183986a82d901edf919] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=33.0 K 2024-11-18T20:08:26,583 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade3b8211b56448fa1b428b040ca7422, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960504384 2024-11-18T20:08:26,583 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4cc6ccfd5e410f8b79d7b76d2a7a29, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960504384 2024-11-18T20:08:26,584 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 878027fe0e2e437994787efdacefeb2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960505312 2024-11-18T20:08:26,585 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting df6b41aa51a44725909f6fe7cdbf4d46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960505312 2024-11-18T20:08:26,585 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d223610c204183986a82d901edf919, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731960505597 2024-11-18T20:08:26,586 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06ef3de4be24624b018f6e2cb473947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731960505597 2024-11-18T20:08:26,621 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#18 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:26,622 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/b07d9761ec7440f08222b1b034496824 is 50, key is test_row_0/B:col10/1731960505597/Put/seqid=0 2024-11-18T20:08:26,624 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#19 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:26,625 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/bddb56e2c5f34c51ac700ee3796c6261 is 50, key is test_row_0/A:col10/1731960505597/Put/seqid=0 2024-11-18T20:08:26,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741855_1031 (size=12207) 2024-11-18T20:08:26,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741856_1032 (size=12207) 2024-11-18T20:08:26,750 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:26,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:08:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:26,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fba02b8959ab41e7bb9a854a9094f54e is 50, key is test_row_0/A:col10/1731960506750/Put/seqid=0 2024-11-18T20:08:26,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960566794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960566792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960566796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960566798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741857_1033 (size=14341) 2024-11-18T20:08:26,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fba02b8959ab41e7bb9a854a9094f54e 2024-11-18T20:08:26,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/66628fff626a43a29cfabdc00350252d is 50, key is test_row_0/B:col10/1731960506750/Put/seqid=0 2024-11-18T20:08:26,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741858_1034 (size=12001) 2024-11-18T20:08:26,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960566913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960566913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960566913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:26,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960566913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,083 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/bddb56e2c5f34c51ac700ee3796c6261 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bddb56e2c5f34c51ac700ee3796c6261 2024-11-18T20:08:27,088 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/b07d9761ec7440f08222b1b034496824 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/b07d9761ec7440f08222b1b034496824 2024-11-18T20:08:27,096 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T20:08:27,097 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T20:08:27,099 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:08:27,099 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T20:08:27,100 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T20:08:27,100 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T20:08:27,101 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:08:27,102 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:08:27,104 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into b07d9761ec7440f08222b1b034496824(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:27,104 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into bddb56e2c5f34c51ac700ee3796c6261(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:27,104 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:27,104 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960506579; duration=0sec 2024-11-18T20:08:27,104 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:27,105 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960506579; duration=0sec 2024-11-18T20:08:27,106 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:27,106 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:27,106 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:27,105 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:27,108 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:27,109 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:27,110 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:27,110 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:27,110 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1f085217667c4e96b8347bb53ea22b05, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/516d98c335fb468f96b9bf86349efb19, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/95adccf8e77643aab90ca613bb0ff395] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=33.0 K 2024-11-18T20:08:27,111 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f085217667c4e96b8347bb53ea22b05, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960504384 2024-11-18T20:08:27,112 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 516d98c335fb468f96b9bf86349efb19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960505312 2024-11-18T20:08:27,112 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 95adccf8e77643aab90ca613bb0ff395, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731960505597 2024-11-18T20:08:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960567124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960567133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960567133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960567133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,145 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#22 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,147 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e2d472209334937883579a52502c179 is 50, key is test_row_0/C:col10/1731960505597/Put/seqid=0 2024-11-18T20:08:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741859_1035 (size=12207) 2024-11-18T20:08:27,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/66628fff626a43a29cfabdc00350252d 2024-11-18T20:08:27,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/58256f68fadd453baef9819b1cdc0c3f is 50, key is test_row_0/C:col10/1731960506750/Put/seqid=0 2024-11-18T20:08:27,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741860_1036 (size=12001) 2024-11-18T20:08:27,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/58256f68fadd453baef9819b1cdc0c3f 2024-11-18T20:08:27,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fba02b8959ab41e7bb9a854a9094f54e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fba02b8959ab41e7bb9a854a9094f54e 2024-11-18T20:08:27,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fba02b8959ab41e7bb9a854a9094f54e, entries=200, sequenceid=117, filesize=14.0 K 2024-11-18T20:08:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/66628fff626a43a29cfabdc00350252d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/66628fff626a43a29cfabdc00350252d 2024-11-18T20:08:27,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/66628fff626a43a29cfabdc00350252d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:08:27,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/58256f68fadd453baef9819b1cdc0c3f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/58256f68fadd453baef9819b1cdc0c3f 2024-11-18T20:08:27,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/58256f68fadd453baef9819b1cdc0c3f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:08:27,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for b989f3762007617cfc5051d63359b50d in 665ms, sequenceid=117, compaction requested=false 2024-11-18T20:08:27,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:27,436 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:27,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:08:27,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81fdaaad17714da39791bdfa2ff7c4f8 is 50, key is test_row_0/A:col10/1731960506794/Put/seqid=0 2024-11-18T20:08:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741861_1037 (size=14391) 2024-11-18T20:08:27,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81fdaaad17714da39791bdfa2ff7c4f8 2024-11-18T20:08:27,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/8f21ff1c5f2e4a389a8f13a392e08bd6 is 50, key is test_row_0/B:col10/1731960506794/Put/seqid=0 2024-11-18T20:08:27,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960567520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960567522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960567526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960567521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960567529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,538 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in b989f3762007617cfc5051d63359b50d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741862_1038 (size=12051) 2024-11-18T20:08:27,629 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e2d472209334937883579a52502c179 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e2d472209334937883579a52502c179 2024-11-18T20:08:27,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960567632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960567643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960567644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960567645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,655 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 9e2d472209334937883579a52502c179(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:27,655 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:27,655 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960506579; duration=0sec 2024-11-18T20:08:27,655 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,655 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960567852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960567853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960567853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:08:27,863 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:27,864 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:27,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960567862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-18T20:08:27,870 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:27,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:27,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:08:27,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/8f21ff1c5f2e4a389a8f13a392e08bd6 2024-11-18T20:08:27,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e1da25de104489999d9eb297079a974 is 50, key is test_row_0/C:col10/1731960506794/Put/seqid=0 2024-11-18T20:08:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:08:28,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741863_1039 (size=12051) 2024-11-18T20:08:28,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e1da25de104489999d9eb297079a974 2024-11-18T20:08:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81fdaaad17714da39791bdfa2ff7c4f8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81fdaaad17714da39791bdfa2ff7c4f8 2024-11-18T20:08:28,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:08:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81fdaaad17714da39791bdfa2ff7c4f8, entries=200, sequenceid=131, filesize=14.1 K 2024-11-18T20:08:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/8f21ff1c5f2e4a389a8f13a392e08bd6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/8f21ff1c5f2e4a389a8f13a392e08bd6 2024-11-18T20:08:28,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/8f21ff1c5f2e4a389a8f13a392e08bd6, entries=150, sequenceid=131, filesize=11.8 K 2024-11-18T20:08:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e1da25de104489999d9eb297079a974 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e1da25de104489999d9eb297079a974 2024-11-18T20:08:28,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e1da25de104489999d9eb297079a974, entries=150, sequenceid=131, filesize=11.8 K 2024-11-18T20:08:28,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b989f3762007617cfc5051d63359b50d in 626ms, sequenceid=131, compaction requested=true 2024-11-18T20:08:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:28,063 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:08:28,065 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:28,066 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:28,066 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:28,066 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,066 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bddb56e2c5f34c51ac700ee3796c6261, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fba02b8959ab41e7bb9a854a9094f54e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81fdaaad17714da39791bdfa2ff7c4f8] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=40.0 K 2024-11-18T20:08:28,068 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting bddb56e2c5f34c51ac700ee3796c6261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731960505312 2024-11-18T20:08:28,070 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:28,070 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:28,070 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,070 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/b07d9761ec7440f08222b1b034496824, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/66628fff626a43a29cfabdc00350252d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/8f21ff1c5f2e4a389a8f13a392e08bd6] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=35.4 K 2024-11-18T20:08:28,071 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting fba02b8959ab41e7bb9a854a9094f54e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960506402 2024-11-18T20:08:28,072 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b07d9761ec7440f08222b1b034496824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731960505312 2024-11-18T20:08:28,072 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fdaaad17714da39791bdfa2ff7c4f8, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731960506789 2024-11-18T20:08:28,074 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66628fff626a43a29cfabdc00350252d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960506419 2024-11-18T20:08:28,075 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f21ff1c5f2e4a389a8f13a392e08bd6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731960506791 2024-11-18T20:08:28,095 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:28,096 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/aa4c80855dc3414c9d57fe06b4765851 is 50, key is test_row_0/A:col10/1731960506794/Put/seqid=0 2024-11-18T20:08:28,106 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:28,107 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/64e758bf4fbf4bb3aebfbc4eebf28f3e is 50, key is test_row_0/B:col10/1731960506794/Put/seqid=0 2024-11-18T20:08:28,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741864_1040 (size=12359) 2024-11-18T20:08:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741865_1041 (size=12359) 2024-11-18T20:08:28,155 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/64e758bf4fbf4bb3aebfbc4eebf28f3e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/64e758bf4fbf4bb3aebfbc4eebf28f3e 2024-11-18T20:08:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:28,172 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:28,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:08:28,174 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 64e758bf4fbf4bb3aebfbc4eebf28f3e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:28,174 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:28,174 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960508063; duration=0sec 2024-11-18T20:08:28,177 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:28,177 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:28,177 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:28,179 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:28,179 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:28,179 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,179 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e2d472209334937883579a52502c179, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/58256f68fadd453baef9819b1cdc0c3f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e1da25de104489999d9eb297079a974] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=35.4 K 2024-11-18T20:08:28,180 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e2d472209334937883579a52502c179, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731960505312 2024-11-18T20:08:28,181 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58256f68fadd453baef9819b1cdc0c3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960506419 2024-11-18T20:08:28,182 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e1da25de104489999d9eb297079a974, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731960506791 2024-11-18T20:08:28,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:08:28,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:28,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/92a44f53bcf34fbaa73475ddaaff2fa8 is 50, key is test_row_0/A:col10/1731960507521/Put/seqid=0 2024-11-18T20:08:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:08:28,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960568200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960568205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960568207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960568211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,238 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:28,239 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/140eea8321cd432180151d9f2a133fb4 is 50, key is test_row_0/C:col10/1731960506794/Put/seqid=0 2024-11-18T20:08:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741866_1042 (size=12151) 2024-11-18T20:08:28,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/92a44f53bcf34fbaa73475ddaaff2fa8 2024-11-18T20:08:28,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/c7e9cfe761474868849deb985a7998bb is 50, key is test_row_0/B:col10/1731960507521/Put/seqid=0 2024-11-18T20:08:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741867_1043 (size=12359) 2024-11-18T20:08:28,304 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/140eea8321cd432180151d9f2a133fb4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/140eea8321cd432180151d9f2a133fb4 2024-11-18T20:08:28,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741868_1044 (size=12151) 2024-11-18T20:08:28,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/c7e9cfe761474868849deb985a7998bb 2024-11-18T20:08:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960568313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960568323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,326 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 140eea8321cd432180151d9f2a133fb4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:28,326 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:28,327 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960508064; duration=0sec 2024-11-18T20:08:28,327 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:28,327 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960568325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960568325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/f63faef58c7c486397324b65810accfa is 50, key is test_row_0/C:col10/1731960507521/Put/seqid=0 2024-11-18T20:08:28,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:08:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:28,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:28,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741869_1045 (size=12151) 2024-11-18T20:08:28,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/f63faef58c7c486397324b65810accfa 2024-11-18T20:08:28,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/92a44f53bcf34fbaa73475ddaaff2fa8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/92a44f53bcf34fbaa73475ddaaff2fa8 2024-11-18T20:08:28,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/92a44f53bcf34fbaa73475ddaaff2fa8, entries=150, sequenceid=159, filesize=11.9 K 2024-11-18T20:08:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/c7e9cfe761474868849deb985a7998bb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/c7e9cfe761474868849deb985a7998bb 2024-11-18T20:08:28,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/c7e9cfe761474868849deb985a7998bb, entries=150, sequenceid=159, filesize=11.9 K 2024-11-18T20:08:28,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/f63faef58c7c486397324b65810accfa as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/f63faef58c7c486397324b65810accfa 2024-11-18T20:08:28,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/f63faef58c7c486397324b65810accfa, entries=150, sequenceid=159, filesize=11.9 K 2024-11-18T20:08:28,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for b989f3762007617cfc5051d63359b50d in 264ms, sequenceid=159, compaction requested=false 2024-11-18T20:08:28,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:28,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:08:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:28,511 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:08:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:28,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ae4fb4bb7f374639a05a1f539fee3a42 is 50, key is test_row_0/A:col10/1731960508204/Put/seqid=0 2024-11-18T20:08:28,551 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/aa4c80855dc3414c9d57fe06b4765851 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/aa4c80855dc3414c9d57fe06b4765851 2024-11-18T20:08:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741870_1046 (size=12151) 2024-11-18T20:08:28,573 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into aa4c80855dc3414c9d57fe06b4765851(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:28,573 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:28,573 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960508063; duration=0sec 2024-11-18T20:08:28,574 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:28,574 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960568602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960568603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960568605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960568608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960568713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960568722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960568722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960568724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960568923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960568932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960568932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960568935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:28,964 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ae4fb4bb7f374639a05a1f539fee3a42 2024-11-18T20:08:28,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/888c7ba913df488ca1cd95618ee64d3d is 50, key is test_row_0/B:col10/1731960508204/Put/seqid=0 2024-11-18T20:08:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:08:29,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741871_1047 (size=12151) 2024-11-18T20:08:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960569235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960569242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960569242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960569252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,433 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/888c7ba913df488ca1cd95618ee64d3d 2024-11-18T20:08:29,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/c31a8c9cf62a483b9c1b38dfb71b5dd3 is 50, key is test_row_0/C:col10/1731960508204/Put/seqid=0 2024-11-18T20:08:29,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741872_1048 (size=12151) 2024-11-18T20:08:29,514 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/c31a8c9cf62a483b9c1b38dfb71b5dd3 2024-11-18T20:08:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ae4fb4bb7f374639a05a1f539fee3a42 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ae4fb4bb7f374639a05a1f539fee3a42 2024-11-18T20:08:29,535 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ae4fb4bb7f374639a05a1f539fee3a42, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:08:29,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/888c7ba913df488ca1cd95618ee64d3d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/888c7ba913df488ca1cd95618ee64d3d 2024-11-18T20:08:29,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960569552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,556 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in b989f3762007617cfc5051d63359b50d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,566 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/888c7ba913df488ca1cd95618ee64d3d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:08:29,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/c31a8c9cf62a483b9c1b38dfb71b5dd3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/c31a8c9cf62a483b9c1b38dfb71b5dd3 2024-11-18T20:08:29,577 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/c31a8c9cf62a483b9c1b38dfb71b5dd3, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:08:29,578 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b989f3762007617cfc5051d63359b50d in 1067ms, sequenceid=171, compaction requested=true 2024-11-18T20:08:29,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:29,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:29,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-18T20:08:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-18T20:08:29,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T20:08:29,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-11-18T20:08:29,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7250 sec 2024-11-18T20:08:29,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:29,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:08:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:29,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/6916b77f3f8b46ccb1fe7c4dd44711b7 is 50, key is test_row_0/A:col10/1731960509754/Put/seqid=0 2024-11-18T20:08:29,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960569772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960569776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960569776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960569777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741873_1049 (size=16931) 2024-11-18T20:08:29,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/6916b77f3f8b46ccb1fe7c4dd44711b7 2024-11-18T20:08:29,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0c93f5ba679c4082964df4a22332b2b6 is 50, key is test_row_0/B:col10/1731960509754/Put/seqid=0 2024-11-18T20:08:29,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741874_1050 (size=12151) 2024-11-18T20:08:29,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0c93f5ba679c4082964df4a22332b2b6 2024-11-18T20:08:29,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/76a4685436894c3e8eeb5b688009d689 is 50, key is test_row_0/C:col10/1731960509754/Put/seqid=0 2024-11-18T20:08:29,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960569893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960569893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960569894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:29,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741875_1051 (size=12151) 2024-11-18T20:08:29,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/76a4685436894c3e8eeb5b688009d689 2024-11-18T20:08:29,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/6916b77f3f8b46ccb1fe7c4dd44711b7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/6916b77f3f8b46ccb1fe7c4dd44711b7 2024-11-18T20:08:29,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/6916b77f3f8b46ccb1fe7c4dd44711b7, entries=250, sequenceid=199, filesize=16.5 K 2024-11-18T20:08:29,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0c93f5ba679c4082964df4a22332b2b6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0c93f5ba679c4082964df4a22332b2b6 2024-11-18T20:08:29,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0c93f5ba679c4082964df4a22332b2b6, entries=150, sequenceid=199, filesize=11.9 K 2024-11-18T20:08:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/76a4685436894c3e8eeb5b688009d689 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/76a4685436894c3e8eeb5b688009d689 2024-11-18T20:08:30,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/76a4685436894c3e8eeb5b688009d689, entries=150, sequenceid=199, filesize=11.9 K 2024-11-18T20:08:30,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b989f3762007617cfc5051d63359b50d in 257ms, sequenceid=199, compaction requested=true 2024-11-18T20:08:30,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:30,014 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:30,014 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:30,017 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:30,017 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:30,017 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,018 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/aa4c80855dc3414c9d57fe06b4765851, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/92a44f53bcf34fbaa73475ddaaff2fa8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ae4fb4bb7f374639a05a1f539fee3a42, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/6916b77f3f8b46ccb1fe7c4dd44711b7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=52.3 K 2024-11-18T20:08:30,018 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:30,018 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:30,018 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,018 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/64e758bf4fbf4bb3aebfbc4eebf28f3e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/c7e9cfe761474868849deb985a7998bb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/888c7ba913df488ca1cd95618ee64d3d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0c93f5ba679c4082964df4a22332b2b6] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=47.7 K 2024-11-18T20:08:30,019 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e758bf4fbf4bb3aebfbc4eebf28f3e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731960506791 2024-11-18T20:08:30,020 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4c80855dc3414c9d57fe06b4765851, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731960506791 2024-11-18T20:08:30,021 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e9cfe761474868849deb985a7998bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731960507512 2024-11-18T20:08:30,021 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 92a44f53bcf34fbaa73475ddaaff2fa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731960507512 2024-11-18T20:08:30,022 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888c7ba913df488ca1cd95618ee64d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960508174 2024-11-18T20:08:30,023 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4fb4bb7f374639a05a1f539fee3a42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960508174 2024-11-18T20:08:30,024 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c93f5ba679c4082964df4a22332b2b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960508589 2024-11-18T20:08:30,025 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6916b77f3f8b46ccb1fe7c4dd44711b7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960508589 2024-11-18T20:08:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:08:30,033 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:30,034 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-18T20:08:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:30,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:30,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:30,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:30,066 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:30,067 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0ff3e353399d4501b2f62e07899dfc2e is 50, key is test_row_0/B:col10/1731960509754/Put/seqid=0 2024-11-18T20:08:30,073 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:30,074 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/857d16faa05a4fb48111f2ad167a1b2c is 50, key is test_row_0/A:col10/1731960509754/Put/seqid=0 2024-11-18T20:08:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:30,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:30,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:30,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741876_1052 (size=12595) 2024-11-18T20:08:30,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/5f23ce75cf004bc68b2e726d23112e45 is 50, key is test_row_0/A:col10/1731960510105/Put/seqid=0 2024-11-18T20:08:30,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741877_1053 (size=12595) 2024-11-18T20:08:30,134 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0ff3e353399d4501b2f62e07899dfc2e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0ff3e353399d4501b2f62e07899dfc2e 2024-11-18T20:08:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:30,144 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/857d16faa05a4fb48111f2ad167a1b2c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/857d16faa05a4fb48111f2ad167a1b2c 2024-11-18T20:08:30,144 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 0ff3e353399d4501b2f62e07899dfc2e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:30,144 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:30,144 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=12, startTime=1731960510014; duration=0sec 2024-11-18T20:08:30,145 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:30,145 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:30,145 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:30,148 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:30,148 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:30,148 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,148 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/140eea8321cd432180151d9f2a133fb4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/f63faef58c7c486397324b65810accfa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/c31a8c9cf62a483b9c1b38dfb71b5dd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/76a4685436894c3e8eeb5b688009d689] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=47.7 K 2024-11-18T20:08:30,149 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140eea8321cd432180151d9f2a133fb4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731960506791 2024-11-18T20:08:30,149 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f63faef58c7c486397324b65810accfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731960507512 2024-11-18T20:08:30,150 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31a8c9cf62a483b9c1b38dfb71b5dd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960508174 2024-11-18T20:08:30,151 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a4685436894c3e8eeb5b688009d689, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960508589 2024-11-18T20:08:30,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741878_1054 (size=16931) 2024-11-18T20:08:30,158 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into 857d16faa05a4fb48111f2ad167a1b2c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:30,158 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:30,158 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=12, startTime=1731960510013; duration=0sec 2024-11-18T20:08:30,158 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:30,158 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:30,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/5f23ce75cf004bc68b2e726d23112e45 2024-11-18T20:08:30,180 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:30,181 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/7288633d79924a1994850606b5ab4825 is 50, key is test_row_0/C:col10/1731960509754/Put/seqid=0 2024-11-18T20:08:30,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/1270e12a09d8418b9e22acb60ed0a704 is 50, key is test_row_0/B:col10/1731960510105/Put/seqid=0 2024-11-18T20:08:30,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960570213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960570220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960570221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741879_1055 (size=12595) 2024-11-18T20:08:30,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741880_1056 (size=12151) 2024-11-18T20:08:30,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960570322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960570332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960570332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:30,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:30,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960570532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960570542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960570542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/1270e12a09d8418b9e22acb60ed0a704 2024-11-18T20:08:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:30,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:30,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:30,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,671 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/7288633d79924a1994850606b5ab4825 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/7288633d79924a1994850606b5ab4825 2024-11-18T20:08:30,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e709ddb6bda43698061553d65726621 is 50, key is test_row_0/C:col10/1731960510105/Put/seqid=0 2024-11-18T20:08:30,683 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 7288633d79924a1994850606b5ab4825(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:30,683 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:30,684 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=12, startTime=1731960510014; duration=0sec 2024-11-18T20:08:30,684 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:30,684 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:30,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741881_1057 (size=12151) 2024-11-18T20:08:30,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e709ddb6bda43698061553d65726621 2024-11-18T20:08:30,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/5f23ce75cf004bc68b2e726d23112e45 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f23ce75cf004bc68b2e726d23112e45 2024-11-18T20:08:30,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f23ce75cf004bc68b2e726d23112e45, entries=250, sequenceid=210, filesize=16.5 K 2024-11-18T20:08:30,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/1270e12a09d8418b9e22acb60ed0a704 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1270e12a09d8418b9e22acb60ed0a704 2024-11-18T20:08:30,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1270e12a09d8418b9e22acb60ed0a704, entries=150, sequenceid=210, filesize=11.9 K 2024-11-18T20:08:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/9e709ddb6bda43698061553d65726621 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e709ddb6bda43698061553d65726621 2024-11-18T20:08:30,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e709ddb6bda43698061553d65726621, entries=150, sequenceid=210, filesize=11.9 K 2024-11-18T20:08:30,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b989f3762007617cfc5051d63359b50d in 672ms, sequenceid=210, compaction requested=false 2024-11-18T20:08:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:30,796 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:30,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:08:30,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/b6fb4b04022a4964bd286292e5617052 is 50, key is test_row_0/A:col10/1731960510215/Put/seqid=0 2024-11-18T20:08:30,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:30,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:30,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960570831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960570842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960570852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960570852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741882_1058 (size=14541) 2024-11-18T20:08:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960570942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:30,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:30,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960571152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:31,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/b6fb4b04022a4964bd286292e5617052 2024-11-18T20:08:31,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:31,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0d75316fd5904cc79a6a41e098058ad7 is 50, key is test_row_0/B:col10/1731960510215/Put/seqid=0 2024-11-18T20:08:31,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741883_1059 (size=12151) 2024-11-18T20:08:31,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960571353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960571362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960571363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:31,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960571462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:31,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0d75316fd5904cc79a6a41e098058ad7 2024-11-18T20:08:31,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/225bac3a6b8e4d4194748f5b0c226873 is 50, key is test_row_0/C:col10/1731960510215/Put/seqid=0 2024-11-18T20:08:31,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741884_1060 (size=12151) 2024-11-18T20:08:31,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:31,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:31,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960571972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:32,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/225bac3a6b8e4d4194748f5b0c226873 2024-11-18T20:08:32,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/b6fb4b04022a4964bd286292e5617052 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b6fb4b04022a4964bd286292e5617052 2024-11-18T20:08:32,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b6fb4b04022a4964bd286292e5617052, entries=200, sequenceid=240, filesize=14.2 K 2024-11-18T20:08:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0d75316fd5904cc79a6a41e098058ad7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0d75316fd5904cc79a6a41e098058ad7 2024-11-18T20:08:32,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0d75316fd5904cc79a6a41e098058ad7, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T20:08:32,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/225bac3a6b8e4d4194748f5b0c226873 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/225bac3a6b8e4d4194748f5b0c226873 2024-11-18T20:08:32,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:32,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:32,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/225bac3a6b8e4d4194748f5b0c226873, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T20:08:32,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b989f3762007617cfc5051d63359b50d in 1467ms, sequenceid=240, compaction requested=true 2024-11-18T20:08:32,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:32,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:32,265 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:32,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:32,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:32,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:32,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:32,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:32,265 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:32,267 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:32,267 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:32,267 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,267 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/857d16faa05a4fb48111f2ad167a1b2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f23ce75cf004bc68b2e726d23112e45, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b6fb4b04022a4964bd286292e5617052] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=43.0 K 2024-11-18T20:08:32,268 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:32,268 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 857d16faa05a4fb48111f2ad167a1b2c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960508589 2024-11-18T20:08:32,268 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:32,268 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,268 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0ff3e353399d4501b2f62e07899dfc2e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1270e12a09d8418b9e22acb60ed0a704, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0d75316fd5904cc79a6a41e098058ad7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=36.0 K 2024-11-18T20:08:32,269 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f23ce75cf004bc68b2e726d23112e45, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731960509770 2024-11-18T20:08:32,270 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff3e353399d4501b2f62e07899dfc2e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960508589 2024-11-18T20:08:32,270 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6fb4b04022a4964bd286292e5617052, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960510209 2024-11-18T20:08:32,271 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1270e12a09d8418b9e22acb60ed0a704, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731960509775 2024-11-18T20:08:32,271 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d75316fd5904cc79a6a41e098058ad7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960510215 2024-11-18T20:08:32,295 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:32,296 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/01aaf077a11f48b5b4e2f50faa9f84f0 is 50, key is test_row_0/A:col10/1731960510215/Put/seqid=0 2024-11-18T20:08:32,306 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:32,307 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f034dd7e1fd9492b987997a58321440d is 50, key is test_row_0/B:col10/1731960510215/Put/seqid=0 2024-11-18T20:08:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741886_1062 (size=12697) 2024-11-18T20:08:32,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741885_1061 (size=12697) 2024-11-18T20:08:32,352 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f034dd7e1fd9492b987997a58321440d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f034dd7e1fd9492b987997a58321440d 2024-11-18T20:08:32,376 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/01aaf077a11f48b5b4e2f50faa9f84f0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/01aaf077a11f48b5b4e2f50faa9f84f0 2024-11-18T20:08:32,380 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:32,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:32,401 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into f034dd7e1fd9492b987997a58321440d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:32,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/073229826af94a6cbbfaad968717bd18 is 50, key is test_row_0/A:col10/1731960512379/Put/seqid=0 2024-11-18T20:08:32,401 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:32,401 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960512265; duration=0sec 2024-11-18T20:08:32,401 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:32,401 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:32,402 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:32,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:32,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,402 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into 01aaf077a11f48b5b4e2f50faa9f84f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:32,402 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:32,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:32,402 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960512264; duration=0sec 2024-11-18T20:08:32,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,402 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:32,402 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:32,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,407 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:32,407 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:32,408 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,408 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/7288633d79924a1994850606b5ab4825, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e709ddb6bda43698061553d65726621, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/225bac3a6b8e4d4194748f5b0c226873] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=36.0 K 2024-11-18T20:08:32,409 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7288633d79924a1994850606b5ab4825, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960508589 2024-11-18T20:08:32,409 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e709ddb6bda43698061553d65726621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731960509775 2024-11-18T20:08:32,410 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 225bac3a6b8e4d4194748f5b0c226873, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960510215 2024-11-18T20:08:32,426 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:32,427 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/bd207320389e418489a1639794881927 is 50, key is test_row_0/C:col10/1731960510215/Put/seqid=0 2024-11-18T20:08:32,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741887_1063 (size=9757) 2024-11-18T20:08:32,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/073229826af94a6cbbfaad968717bd18 2024-11-18T20:08:32,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960572453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/a4da0f9258504fac9a795ccbbc9ec322 is 50, key is test_row_0/B:col10/1731960512379/Put/seqid=0 2024-11-18T20:08:32,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960572456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741888_1064 (size=12697) 2024-11-18T20:08:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960572456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,476 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/bd207320389e418489a1639794881927 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/bd207320389e418489a1639794881927 2024-11-18T20:08:32,486 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into bd207320389e418489a1639794881927(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:32,486 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:32,486 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960512265; duration=0sec 2024-11-18T20:08:32,487 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:32,487 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741889_1065 (size=9757) 2024-11-18T20:08:32,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/a4da0f9258504fac9a795ccbbc9ec322 2024-11-18T20:08:32,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b2e4dd4aa848466182a5f4a51bfd0d39 is 50, key is test_row_0/C:col10/1731960512379/Put/seqid=0 2024-11-18T20:08:32,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741890_1066 (size=9757) 2024-11-18T20:08:32,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960572573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960572574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960572573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960572783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960572783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960572783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:32,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:32,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b2e4dd4aa848466182a5f4a51bfd0d39 2024-11-18T20:08:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/073229826af94a6cbbfaad968717bd18 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/073229826af94a6cbbfaad968717bd18 2024-11-18T20:08:32,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/073229826af94a6cbbfaad968717bd18, entries=100, sequenceid=252, filesize=9.5 K 2024-11-18T20:08:32,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/a4da0f9258504fac9a795ccbbc9ec322 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/a4da0f9258504fac9a795ccbbc9ec322 2024-11-18T20:08:32,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960572983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:32,996 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in b989f3762007617cfc5051d63359b50d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:32,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/a4da0f9258504fac9a795ccbbc9ec322, entries=100, sequenceid=252, filesize=9.5 K 2024-11-18T20:08:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b2e4dd4aa848466182a5f4a51bfd0d39 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b2e4dd4aa848466182a5f4a51bfd0d39 2024-11-18T20:08:33,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b2e4dd4aa848466182a5f4a51bfd0d39, entries=100, sequenceid=252, filesize=9.5 K 2024-11-18T20:08:33,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for b989f3762007617cfc5051d63359b50d in 676ms, sequenceid=252, compaction requested=false 2024-11-18T20:08:33,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:33,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:08:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:33,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ffb275be42d14576b21cdb1e3553d1e6 is 50, key is test_row_0/A:col10/1731960513098/Put/seqid=0 2024-11-18T20:08:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741891_1067 (size=17181) 2024-11-18T20:08:33,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ffb275be42d14576b21cdb1e3553d1e6 2024-11-18T20:08:33,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f42f129d1720405eb3a846748e9c7d29 is 50, key is test_row_0/B:col10/1731960513098/Put/seqid=0 2024-11-18T20:08:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960573179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960573180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960573180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741892_1068 (size=12301) 2024-11-18T20:08:33,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:33,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:33,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960573292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960573292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960573292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960573504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960573504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960573506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:33,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:33,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f42f129d1720405eb3a846748e9c7d29 2024-11-18T20:08:33,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960573601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,605 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in b989f3762007617cfc5051d63359b50d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/3176a5eecef84b638d75a34ac21b2127 is 50, key is test_row_0/C:col10/1731960513098/Put/seqid=0 2024-11-18T20:08:33,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741893_1069 (size=12301) 2024-11-18T20:08:33,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/3176a5eecef84b638d75a34ac21b2127 2024-11-18T20:08:33,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ffb275be42d14576b21cdb1e3553d1e6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ffb275be42d14576b21cdb1e3553d1e6 2024-11-18T20:08:33,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ffb275be42d14576b21cdb1e3553d1e6, entries=250, sequenceid=283, filesize=16.8 K 2024-11-18T20:08:33,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f42f129d1720405eb3a846748e9c7d29 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f42f129d1720405eb3a846748e9c7d29 2024-11-18T20:08:33,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f42f129d1720405eb3a846748e9c7d29, entries=150, sequenceid=283, filesize=12.0 K 2024-11-18T20:08:33,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/3176a5eecef84b638d75a34ac21b2127 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/3176a5eecef84b638d75a34ac21b2127 2024-11-18T20:08:33,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/3176a5eecef84b638d75a34ac21b2127, entries=150, sequenceid=283, filesize=12.0 K 2024-11-18T20:08:33,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b989f3762007617cfc5051d63359b50d in 592ms, sequenceid=283, compaction requested=true 2024-11-18T20:08:33,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:33,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:33,692 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:33,692 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:33,693 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:33,693 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:33,693 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:33,693 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:33,693 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,693 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,693 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f034dd7e1fd9492b987997a58321440d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/a4da0f9258504fac9a795ccbbc9ec322, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f42f129d1720405eb3a846748e9c7d29] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=33.9 K 2024-11-18T20:08:33,693 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/01aaf077a11f48b5b4e2f50faa9f84f0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/073229826af94a6cbbfaad968717bd18, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ffb275be42d14576b21cdb1e3553d1e6] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=38.7 K 2024-11-18T20:08:33,693 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f034dd7e1fd9492b987997a58321440d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960510215 2024-11-18T20:08:33,694 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01aaf077a11f48b5b4e2f50faa9f84f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960510215 2024-11-18T20:08:33,694 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073229826af94a6cbbfaad968717bd18, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731960512369 2024-11-18T20:08:33,694 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a4da0f9258504fac9a795ccbbc9ec322, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731960512369 2024-11-18T20:08:33,694 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f42f129d1720405eb3a846748e9c7d29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731960512440 2024-11-18T20:08:33,695 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb275be42d14576b21cdb1e3553d1e6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731960512440 2024-11-18T20:08:33,725 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:33,726 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/2007b52bcad6495993288223c8b7421a is 50, key is test_row_0/B:col10/1731960513098/Put/seqid=0 2024-11-18T20:08:33,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:08:33,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:08:33,747 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:33,748 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/b522494cf8d24175a231375a554f0cdd is 50, key is test_row_0/A:col10/1731960513098/Put/seqid=0 2024-11-18T20:08:33,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741894_1070 (size=12949) 2024-11-18T20:08:33,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e587cdb8a3624c1fb371affe2e29f3c3 is 50, key is test_row_2/A:col10/1731960513126/Put/seqid=0 2024-11-18T20:08:33,786 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/2007b52bcad6495993288223c8b7421a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2007b52bcad6495993288223c8b7421a 2024-11-18T20:08:33,794 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 2007b52bcad6495993288223c8b7421a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:33,794 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:33,794 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960513692; duration=0sec 2024-11-18T20:08:33,794 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:33,794 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:33,795 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741896_1072 (size=7415) 2024-11-18T20:08:33,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741895_1071 (size=12949) 2024-11-18T20:08:33,797 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:33,797 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:33,798 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,798 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/bd207320389e418489a1639794881927, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b2e4dd4aa848466182a5f4a51bfd0d39, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/3176a5eecef84b638d75a34ac21b2127] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=33.9 K 2024-11-18T20:08:33,798 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting bd207320389e418489a1639794881927, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960510215 2024-11-18T20:08:33,799 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e4dd4aa848466182a5f4a51bfd0d39, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731960512369 2024-11-18T20:08:33,799 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3176a5eecef84b638d75a34ac21b2127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731960512440 2024-11-18T20:08:33,807 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/b522494cf8d24175a231375a554f0cdd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b522494cf8d24175a231375a554f0cdd 2024-11-18T20:08:33,811 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e587cdb8a3624c1fb371affe2e29f3c3 2024-11-18T20:08:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:33,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:33,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/1d7d46f882a44c67bb775c7a02720bd3 is 50, key is test_row_2/B:col10/1731960513126/Put/seqid=0 2024-11-18T20:08:33,831 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into b522494cf8d24175a231375a554f0cdd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:33,832 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:33,832 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960513691; duration=0sec 2024-11-18T20:08:33,832 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:33,832 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:33,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741897_1073 (size=7415) 2024-11-18T20:08:33,842 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:33,842 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/26b020f0593a44d3afbb1f50b65527a6 is 50, key is test_row_0/C:col10/1731960513098/Put/seqid=0 2024-11-18T20:08:33,843 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/1d7d46f882a44c67bb775c7a02720bd3 2024-11-18T20:08:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/edd4e42c2e534d57a376541aa9b1c3d8 is 50, key is test_row_2/C:col10/1731960513126/Put/seqid=0 2024-11-18T20:08:33,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741899_1075 (size=7415) 2024-11-18T20:08:33,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741898_1074 (size=12949) 2024-11-18T20:08:33,874 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/edd4e42c2e534d57a376541aa9b1c3d8 2024-11-18T20:08:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e587cdb8a3624c1fb371affe2e29f3c3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e587cdb8a3624c1fb371affe2e29f3c3 2024-11-18T20:08:33,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960573883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960573884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960573885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:33,896 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e587cdb8a3624c1fb371affe2e29f3c3, entries=50, sequenceid=291, filesize=7.2 K 2024-11-18T20:08:33,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/1d7d46f882a44c67bb775c7a02720bd3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1d7d46f882a44c67bb775c7a02720bd3 2024-11-18T20:08:33,908 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1d7d46f882a44c67bb775c7a02720bd3, entries=50, sequenceid=291, filesize=7.2 K 2024-11-18T20:08:33,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/edd4e42c2e534d57a376541aa9b1c3d8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/edd4e42c2e534d57a376541aa9b1c3d8 2024-11-18T20:08:33,926 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/edd4e42c2e534d57a376541aa9b1c3d8, entries=50, sequenceid=291, filesize=7.2 K 2024-11-18T20:08:33,927 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for b989f3762007617cfc5051d63359b50d in 181ms, sequenceid=291, compaction requested=false 2024-11-18T20:08:33,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:33,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:33,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-18T20:08:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-18T20:08:33,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T20:08:33,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8930 sec 2024-11-18T20:08:33,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 3.9060 sec 2024-11-18T20:08:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:33,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:33,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-18T20:08:34,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fbb823655c9444929e0c89952bbb00a3 is 50, key is test_row_0/A:col10/1731960513993/Put/seqid=0 2024-11-18T20:08:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960574004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960574006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960574007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741900_1076 (size=17181) 2024-11-18T20:08:34,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fbb823655c9444929e0c89952bbb00a3 2024-11-18T20:08:34,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/4b3df0425c1045ffb62bf109cbba4b6f is 50, key is test_row_0/B:col10/1731960513993/Put/seqid=0 2024-11-18T20:08:34,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960574112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960574112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741901_1077 (size=12301) 2024-11-18T20:08:34,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/4b3df0425c1045ffb62bf109cbba4b6f 2024-11-18T20:08:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960574123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b41469fc011e4a99bcd1ded0406493e7 is 50, key is test_row_0/C:col10/1731960513993/Put/seqid=0 2024-11-18T20:08:34,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741902_1078 (size=12301) 2024-11-18T20:08:34,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b41469fc011e4a99bcd1ded0406493e7 2024-11-18T20:08:34,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fbb823655c9444929e0c89952bbb00a3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fbb823655c9444929e0c89952bbb00a3 2024-11-18T20:08:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:08:34,212 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:34,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-18T20:08:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:08:34,220 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:34,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:34,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:34,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fbb823655c9444929e0c89952bbb00a3, entries=250, sequenceid=323, filesize=16.8 K 2024-11-18T20:08:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/4b3df0425c1045ffb62bf109cbba4b6f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/4b3df0425c1045ffb62bf109cbba4b6f 2024-11-18T20:08:34,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/4b3df0425c1045ffb62bf109cbba4b6f, entries=150, sequenceid=323, filesize=12.0 K 2024-11-18T20:08:34,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b41469fc011e4a99bcd1ded0406493e7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b41469fc011e4a99bcd1ded0406493e7 2024-11-18T20:08:34,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b41469fc011e4a99bcd1ded0406493e7, entries=150, sequenceid=323, filesize=12.0 K 2024-11-18T20:08:34,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for b989f3762007617cfc5051d63359b50d in 254ms, sequenceid=323, compaction requested=true 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:34,251 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:08:34,252 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:34,252 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:34,252 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,252 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b522494cf8d24175a231375a554f0cdd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e587cdb8a3624c1fb371affe2e29f3c3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fbb823655c9444929e0c89952bbb00a3] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=36.7 K 2024-11-18T20:08:34,254 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b522494cf8d24175a231375a554f0cdd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731960512440 2024-11-18T20:08:34,255 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e587cdb8a3624c1fb371affe2e29f3c3, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960513126 2024-11-18T20:08:34,255 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb823655c9444929e0c89952bbb00a3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960513871 2024-11-18T20:08:34,277 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:34,278 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/f4ed6eafffd7460998d91e6ac8d5f6de is 50, key is test_row_0/A:col10/1731960513993/Put/seqid=0 2024-11-18T20:08:34,288 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/26b020f0593a44d3afbb1f50b65527a6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/26b020f0593a44d3afbb1f50b65527a6 2024-11-18T20:08:34,299 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 26b020f0593a44d3afbb1f50b65527a6(size=12.6 K), total size for store is 31.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:34,299 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:34,299 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960513692; duration=0sec 2024-11-18T20:08:34,299 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:08:34,299 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:34,299 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:34,299 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:34,301 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:34,301 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:34,311 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,311 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2007b52bcad6495993288223c8b7421a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1d7d46f882a44c67bb775c7a02720bd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/4b3df0425c1045ffb62bf109cbba4b6f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=31.9 K 2024-11-18T20:08:34,312 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2007b52bcad6495993288223c8b7421a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731960512440 2024-11-18T20:08:34,313 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7d46f882a44c67bb775c7a02720bd3, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960513126 2024-11-18T20:08:34,313 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3df0425c1045ffb62bf109cbba4b6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960513871 2024-11-18T20:08:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:08:34,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741903_1079 (size=13051) 2024-11-18T20:08:34,333 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:34,334 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/599ea7f2228448e48f3754e19c3ec978 is 50, key is test_row_0/B:col10/1731960513993/Put/seqid=0 2024-11-18T20:08:34,335 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/f4ed6eafffd7460998d91e6ac8d5f6de as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/f4ed6eafffd7460998d91e6ac8d5f6de 2024-11-18T20:08:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:34,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:34,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:34,350 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into f4ed6eafffd7460998d91e6ac8d5f6de(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:34,350 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:34,350 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960514251; duration=0sec 2024-11-18T20:08:34,350 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:34,350 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:34,350 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:34,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/8f5eb178ac534d32b10a4ca2df705c32 is 50, key is test_row_0/A:col10/1731960514336/Put/seqid=0 2024-11-18T20:08:34,353 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:34,353 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:34,353 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,353 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/26b020f0593a44d3afbb1f50b65527a6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/edd4e42c2e534d57a376541aa9b1c3d8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b41469fc011e4a99bcd1ded0406493e7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=31.9 K 2024-11-18T20:08:34,354 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26b020f0593a44d3afbb1f50b65527a6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731960512440 2024-11-18T20:08:34,355 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd4e42c2e534d57a376541aa9b1c3d8, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960513126 2024-11-18T20:08:34,355 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41469fc011e4a99bcd1ded0406493e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960513871 2024-11-18T20:08:34,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741904_1080 (size=13051) 2024-11-18T20:08:34,385 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#69 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:34,386 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/5b8ff1d79f694d6d940f02f503fefd3b is 50, key is test_row_0/C:col10/1731960513993/Put/seqid=0 2024-11-18T20:08:34,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741905_1081 (size=14741) 2024-11-18T20:08:34,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/8f5eb178ac534d32b10a4ca2df705c32 2024-11-18T20:08:34,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/450e7fbedc5c45dfb4a388a31bf7033b is 50, key is test_row_0/B:col10/1731960514336/Put/seqid=0 2024-11-18T20:08:34,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960574413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960574415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960574421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741907_1083 (size=12301) 2024-11-18T20:08:34,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741906_1082 (size=13051) 2024-11-18T20:08:34,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:08:34,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960574533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960574533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960574533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960574742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960574742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960574743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:34,792 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/599ea7f2228448e48f3754e19c3ec978 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/599ea7f2228448e48f3754e19c3ec978 2024-11-18T20:08:34,799 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 599ea7f2228448e48f3754e19c3ec978(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:34,799 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:34,799 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960514251; duration=0sec 2024-11-18T20:08:34,800 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:34,800 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:34,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/450e7fbedc5c45dfb4a388a31bf7033b 2024-11-18T20:08:34,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:08:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,850 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/5b8ff1d79f694d6d940f02f503fefd3b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/5b8ff1d79f694d6d940f02f503fefd3b 2024-11-18T20:08:34,859 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 5b8ff1d79f694d6d940f02f503fefd3b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:34,859 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:34,859 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960514251; duration=0sec 2024-11-18T20:08:34,859 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:34,859 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:34,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/ac09bbb182cb45ca96a1b6f0cd4cb856 is 50, key is test_row_0/C:col10/1731960514336/Put/seqid=0 2024-11-18T20:08:34,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741908_1084 (size=12301) 2024-11-18T20:08:34,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:34,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:35,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960575012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,014 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in b989f3762007617cfc5051d63359b50d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960575052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960575052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960575053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:35,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:35,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/ac09bbb182cb45ca96a1b6f0cd4cb856 2024-11-18T20:08:35,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/8f5eb178ac534d32b10a4ca2df705c32 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8f5eb178ac534d32b10a4ca2df705c32 2024-11-18T20:08:35,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8f5eb178ac534d32b10a4ca2df705c32, entries=200, sequenceid=335, filesize=14.4 K 2024-11-18T20:08:35,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/450e7fbedc5c45dfb4a388a31bf7033b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/450e7fbedc5c45dfb4a388a31bf7033b 2024-11-18T20:08:35,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/450e7fbedc5c45dfb4a388a31bf7033b, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T20:08:35,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/ac09bbb182cb45ca96a1b6f0cd4cb856 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/ac09bbb182cb45ca96a1b6f0cd4cb856 2024-11-18T20:08:35,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/ac09bbb182cb45ca96a1b6f0cd4cb856, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T20:08:35,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b989f3762007617cfc5051d63359b50d in 967ms, sequenceid=335, compaction requested=false 2024-11-18T20:08:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:35,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:08:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:35,318 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:08:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/2a9284166f204faa86f8d2e578829a6f is 50, key is test_row_0/A:col10/1731960514405/Put/seqid=0 2024-11-18T20:08:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:08:35,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741909_1085 (size=12301) 2024-11-18T20:08:35,357 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/2a9284166f204faa86f8d2e578829a6f 2024-11-18T20:08:35,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/75f8018abdb54325a2aa0af187eb3e30 is 50, key is test_row_0/B:col10/1731960514405/Put/seqid=0 2024-11-18T20:08:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741910_1086 (size=12301) 2024-11-18T20:08:35,399 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/75f8018abdb54325a2aa0af187eb3e30 2024-11-18T20:08:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/fc128d79e6c84b56a130bb4d3700e061 is 50, key is test_row_0/C:col10/1731960514405/Put/seqid=0 2024-11-18T20:08:35,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741911_1087 (size=12301) 2024-11-18T20:08:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960575575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960575576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960575577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960575692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960575692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960575695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,836 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/fc128d79e6c84b56a130bb4d3700e061 2024-11-18T20:08:35,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/2a9284166f204faa86f8d2e578829a6f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/2a9284166f204faa86f8d2e578829a6f 2024-11-18T20:08:35,852 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/2a9284166f204faa86f8d2e578829a6f, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:08:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/75f8018abdb54325a2aa0af187eb3e30 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/75f8018abdb54325a2aa0af187eb3e30 2024-11-18T20:08:35,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/75f8018abdb54325a2aa0af187eb3e30, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:08:35,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/fc128d79e6c84b56a130bb4d3700e061 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fc128d79e6c84b56a130bb4d3700e061 2024-11-18T20:08:35,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fc128d79e6c84b56a130bb4d3700e061, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:08:35,871 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for b989f3762007617cfc5051d63359b50d in 553ms, sequenceid=363, compaction requested=true 2024-11-18T20:08:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-18T20:08:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-18T20:08:35,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T20:08:35,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6510 sec 2024-11-18T20:08:35,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6600 sec 2024-11-18T20:08:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:35,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:35,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:08:35,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e7a01981550a46918f0cc4d4bff50481 is 50, key is test_row_0/A:col10/1731960515574/Put/seqid=0 2024-11-18T20:08:35,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741912_1088 (size=14741) 2024-11-18T20:08:35,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e7a01981550a46918f0cc4d4bff50481 2024-11-18T20:08:35,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/54e074b7257146b98326ff8eddecee54 is 50, key is test_row_0/B:col10/1731960515574/Put/seqid=0 2024-11-18T20:08:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960575940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960575941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960575943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:35,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741913_1089 (size=12301) 2024-11-18T20:08:36,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960576052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960576052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960576052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960576262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960576262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960576262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/54e074b7257146b98326ff8eddecee54 2024-11-18T20:08:36,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/72b73fa9083f4d91b8e9b7dbc0596879 is 50, key is test_row_0/C:col10/1731960515574/Put/seqid=0 2024-11-18T20:08:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:08:36,363 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741914_1090 (size=12301) 2024-11-18T20:08:36,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/72b73fa9083f4d91b8e9b7dbc0596879 2024-11-18T20:08:36,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-18T20:08:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:08:36,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:36,372 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:36,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:36,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/e7a01981550a46918f0cc4d4bff50481 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e7a01981550a46918f0cc4d4bff50481 2024-11-18T20:08:36,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e7a01981550a46918f0cc4d4bff50481, entries=200, sequenceid=378, filesize=14.4 K 2024-11-18T20:08:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/54e074b7257146b98326ff8eddecee54 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/54e074b7257146b98326ff8eddecee54 2024-11-18T20:08:36,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/54e074b7257146b98326ff8eddecee54, entries=150, sequenceid=378, filesize=12.0 K 2024-11-18T20:08:36,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/72b73fa9083f4d91b8e9b7dbc0596879 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/72b73fa9083f4d91b8e9b7dbc0596879 2024-11-18T20:08:36,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/72b73fa9083f4d91b8e9b7dbc0596879, entries=150, sequenceid=378, filesize=12.0 K 2024-11-18T20:08:36,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b989f3762007617cfc5051d63359b50d in 513ms, sequenceid=378, compaction requested=true 2024-11-18T20:08:36,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:36,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:36,417 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:36,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:36,417 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:36,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:36,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:36,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:36,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:36,421 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:36,421 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:36,421 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:36,421 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:36,421 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:36,421 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:36,421 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/f4ed6eafffd7460998d91e6ac8d5f6de, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8f5eb178ac534d32b10a4ca2df705c32, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/2a9284166f204faa86f8d2e578829a6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e7a01981550a46918f0cc4d4bff50481] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=53.5 K 2024-11-18T20:08:36,421 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/599ea7f2228448e48f3754e19c3ec978, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/450e7fbedc5c45dfb4a388a31bf7033b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/75f8018abdb54325a2aa0af187eb3e30, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/54e074b7257146b98326ff8eddecee54] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=48.8 K 2024-11-18T20:08:36,422 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4ed6eafffd7460998d91e6ac8d5f6de, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960513871 2024-11-18T20:08:36,422 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f5eb178ac534d32b10a4ca2df705c32, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731960514001 2024-11-18T20:08:36,422 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 599ea7f2228448e48f3754e19c3ec978, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960513871 2024-11-18T20:08:36,423 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a9284166f204faa86f8d2e578829a6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731960514389 2024-11-18T20:08:36,423 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 450e7fbedc5c45dfb4a388a31bf7033b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731960514004 2024-11-18T20:08:36,424 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 75f8018abdb54325a2aa0af187eb3e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731960514389 2024-11-18T20:08:36,425 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a01981550a46918f0cc4d4bff50481, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731960515562 2024-11-18T20:08:36,425 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e074b7257146b98326ff8eddecee54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731960515562 2024-11-18T20:08:36,451 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:36,452 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/75daee86595c431099c4a1d848966ad8 is 50, key is test_row_0/A:col10/1731960515574/Put/seqid=0 2024-11-18T20:08:36,455 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:36,455 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f78b7223162949ab9d2e91b1806cd73b is 50, key is test_row_0/B:col10/1731960515574/Put/seqid=0 2024-11-18T20:08:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:08:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741915_1091 (size=13187) 2024-11-18T20:08:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741916_1092 (size=13187) 2024-11-18T20:08:36,493 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f78b7223162949ab9d2e91b1806cd73b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f78b7223162949ab9d2e91b1806cd73b 2024-11-18T20:08:36,506 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into f78b7223162949ab9d2e91b1806cd73b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:36,506 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:36,506 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=12, startTime=1731960516417; duration=0sec 2024-11-18T20:08:36,506 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:36,506 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:36,506 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:36,509 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:36,509 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:36,509 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:36,509 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/5b8ff1d79f694d6d940f02f503fefd3b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/ac09bbb182cb45ca96a1b6f0cd4cb856, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fc128d79e6c84b56a130bb4d3700e061, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/72b73fa9083f4d91b8e9b7dbc0596879] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=48.8 K 2024-11-18T20:08:36,510 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8ff1d79f694d6d940f02f503fefd3b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960513871 2024-11-18T20:08:36,511 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ac09bbb182cb45ca96a1b6f0cd4cb856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731960514004 2024-11-18T20:08:36,511 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting fc128d79e6c84b56a130bb4d3700e061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731960514389 2024-11-18T20:08:36,512 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b73fa9083f4d91b8e9b7dbc0596879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731960515562 2024-11-18T20:08:36,523 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:36,524 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/19b483cd5d6f48468d329d8d5815f966 is 50, key is test_row_0/C:col10/1731960515574/Put/seqid=0 2024-11-18T20:08:36,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T20:08:36,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:36,533 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:08:36,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/8ccdd4a5d25b42cca5f876899cc39341 is 50, key is test_row_0/A:col10/1731960515941/Put/seqid=0 2024-11-18T20:08:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741917_1093 (size=13187) 2024-11-18T20:08:36,575 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/19b483cd5d6f48468d329d8d5815f966 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/19b483cd5d6f48468d329d8d5815f966 2024-11-18T20:08:36,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741918_1094 (size=12301) 2024-11-18T20:08:36,584 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/8ccdd4a5d25b42cca5f876899cc39341 2024-11-18T20:08:36,586 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 19b483cd5d6f48468d329d8d5815f966(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:36,586 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:36,586 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=12, startTime=1731960516417; duration=0sec 2024-11-18T20:08:36,587 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:36,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:36,588 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:36,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/7cf6c74683cd430f9564970a28e3fd1c is 50, key is test_row_0/B:col10/1731960515941/Put/seqid=0 2024-11-18T20:08:36,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741919_1095 (size=12301) 2024-11-18T20:08:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960576628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960576629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960576631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:08:36,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960576742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960576742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960576742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,880 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/75daee86595c431099c4a1d848966ad8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/75daee86595c431099c4a1d848966ad8 2024-11-18T20:08:36,887 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into 75daee86595c431099c4a1d848966ad8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:36,887 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:36,887 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=12, startTime=1731960516416; duration=0sec 2024-11-18T20:08:36,887 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:36,887 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:36,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960576952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960576959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960576962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:08:37,010 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/7cf6c74683cd430f9564970a28e3fd1c 2024-11-18T20:08:37,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4c1409af7bf64c07a234b17cc503ea9a is 50, key is test_row_0/C:col10/1731960515941/Put/seqid=0 2024-11-18T20:08:37,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741920_1096 (size=12301) 2024-11-18T20:08:37,041 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4c1409af7bf64c07a234b17cc503ea9a 2024-11-18T20:08:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/8ccdd4a5d25b42cca5f876899cc39341 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8ccdd4a5d25b42cca5f876899cc39341 2024-11-18T20:08:37,056 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8ccdd4a5d25b42cca5f876899cc39341, entries=150, sequenceid=400, filesize=12.0 K 2024-11-18T20:08:37,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/7cf6c74683cd430f9564970a28e3fd1c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7cf6c74683cd430f9564970a28e3fd1c 2024-11-18T20:08:37,071 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7cf6c74683cd430f9564970a28e3fd1c, entries=150, sequenceid=400, filesize=12.0 K 2024-11-18T20:08:37,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4c1409af7bf64c07a234b17cc503ea9a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4c1409af7bf64c07a234b17cc503ea9a 2024-11-18T20:08:37,090 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4c1409af7bf64c07a234b17cc503ea9a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-18T20:08:37,100 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b989f3762007617cfc5051d63359b50d in 566ms, sequenceid=400, compaction requested=false 2024-11-18T20:08:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-18T20:08:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-18T20:08:37,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T20:08:37,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 730 msec 2024-11-18T20:08:37,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 739 msec 2024-11-18T20:08:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:37,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:37,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:08:37,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c01c1bec744d4796b9b8882f4d6ee8c1 is 50, key is test_row_0/A:col10/1731960516629/Put/seqid=0 2024-11-18T20:08:37,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741921_1097 (size=12301) 2024-11-18T20:08:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960577316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960577321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960577322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960577432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960577432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960577432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:08:37,503 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:37,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-18T20:08:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:08:37,522 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:37,523 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:37,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:08:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960577642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960577643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960577643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:08:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:37,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c01c1bec744d4796b9b8882f4d6ee8c1 2024-11-18T20:08:37,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f97191c8253b43f49f2c30ed6da1cb2c is 50, key is test_row_0/B:col10/1731960516629/Put/seqid=0 2024-11-18T20:08:37,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741922_1098 (size=12301) 2024-11-18T20:08:37,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f97191c8253b43f49f2c30ed6da1cb2c 2024-11-18T20:08:37,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/66c3086ee12541648322f183c090abac is 50, key is test_row_0/C:col10/1731960516629/Put/seqid=0 2024-11-18T20:08:37,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741923_1099 (size=12301) 2024-11-18T20:08:37,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/66c3086ee12541648322f183c090abac 2024-11-18T20:08:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c01c1bec744d4796b9b8882f4d6ee8c1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c01c1bec744d4796b9b8882f4d6ee8c1 2024-11-18T20:08:37,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c01c1bec744d4796b9b8882f4d6ee8c1, entries=150, sequenceid=418, filesize=12.0 K 2024-11-18T20:08:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/f97191c8253b43f49f2c30ed6da1cb2c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f97191c8253b43f49f2c30ed6da1cb2c 2024-11-18T20:08:37,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f97191c8253b43f49f2c30ed6da1cb2c, entries=150, sequenceid=418, filesize=12.0 K 2024-11-18T20:08:37,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/66c3086ee12541648322f183c090abac as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/66c3086ee12541648322f183c090abac 2024-11-18T20:08:37,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/66c3086ee12541648322f183c090abac, entries=150, sequenceid=418, filesize=12.0 K 2024-11-18T20:08:37,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for b989f3762007617cfc5051d63359b50d in 554ms, sequenceid=418, compaction requested=true 2024-11-18T20:08:37,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:37,820 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:37,824 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:37,828 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:37,828 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:37,828 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,828 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/75daee86595c431099c4a1d848966ad8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8ccdd4a5d25b42cca5f876899cc39341, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c01c1bec744d4796b9b8882f4d6ee8c1] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=36.9 K 2024-11-18T20:08:37,829 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:37,829 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:37,829 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,829 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f78b7223162949ab9d2e91b1806cd73b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7cf6c74683cd430f9564970a28e3fd1c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f97191c8253b43f49f2c30ed6da1cb2c] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=36.9 K 2024-11-18T20:08:37,829 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75daee86595c431099c4a1d848966ad8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731960515562 2024-11-18T20:08:37,829 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f78b7223162949ab9d2e91b1806cd73b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731960515562 2024-11-18T20:08:37,830 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cf6c74683cd430f9564970a28e3fd1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731960515927 2024-11-18T20:08:37,830 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ccdd4a5d25b42cca5f876899cc39341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731960515927 2024-11-18T20:08:37,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f97191c8253b43f49f2c30ed6da1cb2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731960516627 2024-11-18T20:08:37,831 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01c1bec744d4796b9b8882f4d6ee8c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731960516627 2024-11-18T20:08:37,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:08:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,836 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:08:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:08:37,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/d34f974192514051ad174d49b9b56f4b is 50, key is test_row_0/A:col10/1731960517306/Put/seqid=0 2024-11-18T20:08:37,869 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:37,870 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/3ad9648dcb0e4ecfbaa9e580226b8ac2 is 50, key is test_row_0/B:col10/1731960516629/Put/seqid=0 2024-11-18T20:08:37,885 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:37,886 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/4bb566df49384f3caeffd9a791667fff is 50, key is test_row_0/A:col10/1731960516629/Put/seqid=0 2024-11-18T20:08:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741924_1100 (size=12301) 2024-11-18T20:08:37,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741925_1101 (size=13289) 2024-11-18T20:08:37,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741926_1102 (size=13289) 2024-11-18T20:08:37,946 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/4bb566df49384f3caeffd9a791667fff as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/4bb566df49384f3caeffd9a791667fff 2024-11-18T20:08:37,946 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/3ad9648dcb0e4ecfbaa9e580226b8ac2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3ad9648dcb0e4ecfbaa9e580226b8ac2 2024-11-18T20:08:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:37,957 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into 4bb566df49384f3caeffd9a791667fff(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:37,957 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:37,957 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=13, startTime=1731960517820; duration=0sec 2024-11-18T20:08:37,958 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:37,958 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:37,958 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:37,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:37,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:37,967 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:37,967 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/19b483cd5d6f48468d329d8d5815f966, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4c1409af7bf64c07a234b17cc503ea9a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/66c3086ee12541648322f183c090abac] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=36.9 K 2024-11-18T20:08:37,968 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b483cd5d6f48468d329d8d5815f966, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731960515562 2024-11-18T20:08:37,969 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1409af7bf64c07a234b17cc503ea9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731960515927 2024-11-18T20:08:37,969 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 3ad9648dcb0e4ecfbaa9e580226b8ac2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:37,969 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:37,969 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=13, startTime=1731960517824; duration=0sec 2024-11-18T20:08:37,969 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c3086ee12541648322f183c090abac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731960516627 2024-11-18T20:08:37,969 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:37,969 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:37,991 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#90 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:37,991 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/d294af8fb21847629829822bed427850 is 50, key is test_row_0/C:col10/1731960516629/Put/seqid=0 2024-11-18T20:08:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960577997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960577998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960578000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741927_1103 (size=13289) 2024-11-18T20:08:38,021 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/d294af8fb21847629829822bed427850 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d294af8fb21847629829822bed427850 2024-11-18T20:08:38,033 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into d294af8fb21847629829822bed427850(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:38,033 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:38,033 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=13, startTime=1731960517824; duration=0sec 2024-11-18T20:08:38,033 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:38,034 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960578112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960578112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960578112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:08:38,322 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/d34f974192514051ad174d49b9b56f4b 2024-11-18T20:08:38,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960578322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960578322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960578323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/16a8e3c0eeab4e27849e3235fb6e240c is 50, key is test_row_0/B:col10/1731960517306/Put/seqid=0 2024-11-18T20:08:38,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741928_1104 (size=12301) 2024-11-18T20:08:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960578631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960578632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960578632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:08:38,747 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/16a8e3c0eeab4e27849e3235fb6e240c 2024-11-18T20:08:38,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4deeb2488cba4507bc0e5848bb4ab394 is 50, key is test_row_0/C:col10/1731960517306/Put/seqid=0 2024-11-18T20:08:38,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741929_1105 (size=12301) 2024-11-18T20:08:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58086 deadline: 1731960579052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:39,054 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in b989f3762007617cfc5051d63359b50d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:39,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960579142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960579142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960579142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:39,192 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4deeb2488cba4507bc0e5848bb4ab394 2024-11-18T20:08:39,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/d34f974192514051ad174d49b9b56f4b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/d34f974192514051ad174d49b9b56f4b 2024-11-18T20:08:39,208 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/d34f974192514051ad174d49b9b56f4b, entries=150, sequenceid=439, filesize=12.0 K 2024-11-18T20:08:39,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/16a8e3c0eeab4e27849e3235fb6e240c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/16a8e3c0eeab4e27849e3235fb6e240c 2024-11-18T20:08:39,218 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/16a8e3c0eeab4e27849e3235fb6e240c, entries=150, sequenceid=439, filesize=12.0 K 2024-11-18T20:08:39,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4deeb2488cba4507bc0e5848bb4ab394 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4deeb2488cba4507bc0e5848bb4ab394 2024-11-18T20:08:39,227 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4deeb2488cba4507bc0e5848bb4ab394, entries=150, sequenceid=439, filesize=12.0 K 2024-11-18T20:08:39,228 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for b989f3762007617cfc5051d63359b50d in 1391ms, sequenceid=439, compaction requested=false 2024-11-18T20:08:39,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:39,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:39,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-18T20:08:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-18T20:08:39,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T20:08:39,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7060 sec 2024-11-18T20:08:39,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7280 sec 2024-11-18T20:08:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:08:39,673 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:39,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-18T20:08:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:08:39,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:39,680 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:39,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:08:39,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:08:39,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:39,836 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:08:39,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/21a11fdac53a4e6393404dbd157dc99a is 50, key is test_row_0/A:col10/1731960517996/Put/seqid=0 2024-11-18T20:08:39,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741930_1106 (size=12301) 2024-11-18T20:08:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:08:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:40,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960580178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960580180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960580182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/21a11fdac53a4e6393404dbd157dc99a 2024-11-18T20:08:40,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/e34aa8d4a782432c800c44f04115eee1 is 50, key is test_row_0/B:col10/1731960517996/Put/seqid=0 2024-11-18T20:08:40,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741931_1107 (size=12301) 2024-11-18T20:08:40,272 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/e34aa8d4a782432c800c44f04115eee1 2024-11-18T20:08:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960580292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960580292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960580292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b6f5d3b042da494a8b72219f9ae6d3de is 50, key is test_row_0/C:col10/1731960517996/Put/seqid=0 2024-11-18T20:08:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:08:40,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741932_1108 (size=12301) 2024-11-18T20:08:40,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960580502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960580502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960580512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b6f5d3b042da494a8b72219f9ae6d3de 2024-11-18T20:08:40,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/21a11fdac53a4e6393404dbd157dc99a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/21a11fdac53a4e6393404dbd157dc99a 2024-11-18T20:08:40,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/21a11fdac53a4e6393404dbd157dc99a, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T20:08:40,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/e34aa8d4a782432c800c44f04115eee1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e34aa8d4a782432c800c44f04115eee1 2024-11-18T20:08:40,729 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e34aa8d4a782432c800c44f04115eee1, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T20:08:40,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/b6f5d3b042da494a8b72219f9ae6d3de as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b6f5d3b042da494a8b72219f9ae6d3de 2024-11-18T20:08:40,736 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b6f5d3b042da494a8b72219f9ae6d3de, entries=150, sequenceid=458, filesize=12.0 K 2024-11-18T20:08:40,737 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b989f3762007617cfc5051d63359b50d in 901ms, sequenceid=458, compaction requested=true 2024-11-18T20:08:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:40,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-18T20:08:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-18T20:08:40,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T20:08:40,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0590 sec 2024-11-18T20:08:40,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0680 sec 2024-11-18T20:08:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:08:40,812 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:40,813 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:40,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:40,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:08:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-18T20:08:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:08:40,815 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:40,815 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:40,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:40,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c12173e356b244ddbb98137a5a4b121b is 50, key is test_row_0/A:col10/1731960520176/Put/seqid=0 2024-11-18T20:08:40,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741933_1109 (size=14741) 2024-11-18T20:08:40,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960580831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960580834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960580834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:08:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960580942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960580942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960580942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:40,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:08:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:40,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:40,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:08:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:08:41,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960581153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c12173e356b244ddbb98137a5a4b121b 2024-11-18T20:08:41,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/2a44c75247af41d29bb54d30301d2b90 is 50, key is test_row_0/B:col10/1731960520176/Put/seqid=0 2024-11-18T20:08:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741934_1110 (size=12301) 2024-11-18T20:08:41,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/2a44c75247af41d29bb54d30301d2b90 2024-11-18T20:08:41,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/0eaba45f19b248249aede2deef759446 is 50, key is test_row_0/C:col10/1731960520176/Put/seqid=0 2024-11-18T20:08:41,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:08:41,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:41,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741935_1111 (size=12301) 2024-11-18T20:08:41,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/0eaba45f19b248249aede2deef759446 2024-11-18T20:08:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c12173e356b244ddbb98137a5a4b121b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c12173e356b244ddbb98137a5a4b121b 2024-11-18T20:08:41,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c12173e356b244ddbb98137a5a4b121b, entries=200, sequenceid=479, filesize=14.4 K 2024-11-18T20:08:41,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/2a44c75247af41d29bb54d30301d2b90 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2a44c75247af41d29bb54d30301d2b90 2024-11-18T20:08:41,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2a44c75247af41d29bb54d30301d2b90, entries=150, sequenceid=479, filesize=12.0 K 2024-11-18T20:08:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/0eaba45f19b248249aede2deef759446 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/0eaba45f19b248249aede2deef759446 2024-11-18T20:08:41,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/0eaba45f19b248249aede2deef759446, entries=150, sequenceid=479, filesize=12.0 K 2024-11-18T20:08:41,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b989f3762007617cfc5051d63359b50d in 507ms, sequenceid=479, compaction requested=true 2024-11-18T20:08:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:41,320 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:41,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:41,321 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:41,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:41,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:41,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:41,322 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:41,322 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:41,322 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:41,322 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:41,322 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,322 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,323 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/4bb566df49384f3caeffd9a791667fff, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/d34f974192514051ad174d49b9b56f4b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/21a11fdac53a4e6393404dbd157dc99a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c12173e356b244ddbb98137a5a4b121b] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=51.4 K 2024-11-18T20:08:41,323 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3ad9648dcb0e4ecfbaa9e580226b8ac2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/16a8e3c0eeab4e27849e3235fb6e240c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e34aa8d4a782432c800c44f04115eee1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2a44c75247af41d29bb54d30301d2b90] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=49.0 K 2024-11-18T20:08:41,323 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb566df49384f3caeffd9a791667fff, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731960516627 2024-11-18T20:08:41,323 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ad9648dcb0e4ecfbaa9e580226b8ac2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731960516627 2024-11-18T20:08:41,323 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d34f974192514051ad174d49b9b56f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731960517306 2024-11-18T20:08:41,324 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a8e3c0eeab4e27849e3235fb6e240c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731960517306 2024-11-18T20:08:41,324 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a11fdac53a4e6393404dbd157dc99a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731960517989 2024-11-18T20:08:41,324 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e34aa8d4a782432c800c44f04115eee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731960517989 2024-11-18T20:08:41,324 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c12173e356b244ddbb98137a5a4b121b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731960520171 2024-11-18T20:08:41,325 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a44c75247af41d29bb54d30301d2b90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731960520171 2024-11-18T20:08:41,335 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:41,336 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ff8904fde6d540c88de3459552c1d367 is 50, key is test_row_0/A:col10/1731960520176/Put/seqid=0 2024-11-18T20:08:41,336 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:41,336 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0b0bd0dd0f8a473b9f22933a5e54527a is 50, key is test_row_0/B:col10/1731960520176/Put/seqid=0 2024-11-18T20:08:41,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741936_1112 (size=13425) 2024-11-18T20:08:41,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741937_1113 (size=13425) 2024-11-18T20:08:41,354 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/ff8904fde6d540c88de3459552c1d367 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ff8904fde6d540c88de3459552c1d367 2024-11-18T20:08:41,356 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0b0bd0dd0f8a473b9f22933a5e54527a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0b0bd0dd0f8a473b9f22933a5e54527a 2024-11-18T20:08:41,364 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 0b0bd0dd0f8a473b9f22933a5e54527a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:41,364 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into ff8904fde6d540c88de3459552c1d367(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:41,364 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:41,364 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:41,364 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=12, startTime=1731960521321; duration=0sec 2024-11-18T20:08:41,364 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=12, startTime=1731960521320; duration=0sec 2024-11-18T20:08:41,365 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:41,365 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:41,364 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:41,365 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:41,365 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:41,368 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:41,368 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:41,368 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,368 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d294af8fb21847629829822bed427850, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4deeb2488cba4507bc0e5848bb4ab394, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b6f5d3b042da494a8b72219f9ae6d3de, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/0eaba45f19b248249aede2deef759446] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=49.0 K 2024-11-18T20:08:41,369 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d294af8fb21847629829822bed427850, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731960516627 2024-11-18T20:08:41,369 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4deeb2488cba4507bc0e5848bb4ab394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731960517306 2024-11-18T20:08:41,370 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f5d3b042da494a8b72219f9ae6d3de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731960517989 2024-11-18T20:08:41,370 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eaba45f19b248249aede2deef759446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731960520171 2024-11-18T20:08:41,387 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:41,387 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4ee44e538b57440f8a9b34ceee597c8f is 50, key is test_row_0/C:col10/1731960520176/Put/seqid=0 2024-11-18T20:08:41,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741938_1114 (size=13425) 2024-11-18T20:08:41,400 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4ee44e538b57440f8a9b34ceee597c8f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4ee44e538b57440f8a9b34ceee597c8f 2024-11-18T20:08:41,406 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into 4ee44e538b57440f8a9b34ceee597c8f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:41,406 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:41,406 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=12, startTime=1731960521321; duration=0sec 2024-11-18T20:08:41,406 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:41,406 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:41,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:08:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:41,429 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:08:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/5f5d6f39e2de47aea6b9c82450190066 is 50, key is test_row_0/A:col10/1731960520833/Put/seqid=0 2024-11-18T20:08:41,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741939_1115 (size=12301) 2024-11-18T20:08:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:08:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:41,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. as already flushing 2024-11-18T20:08:41,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960581485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960581485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960581485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960581592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960581592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960581602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960581801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960581802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960581812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:41,842 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/5f5d6f39e2de47aea6b9c82450190066 2024-11-18T20:08:41,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/7ac1a1de5fd34a298a4512c58c7cdbc3 is 50, key is test_row_0/B:col10/1731960520833/Put/seqid=0 2024-11-18T20:08:41,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741940_1116 (size=12301) 2024-11-18T20:08:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:08:42,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:42,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960582112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960582112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960582121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:42,258 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/7ac1a1de5fd34a298a4512c58c7cdbc3 2024-11-18T20:08:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/1a8df4db0a8e4b18b525b89a3b816504 is 50, key is test_row_0/C:col10/1731960520833/Put/seqid=0 2024-11-18T20:08:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741941_1117 (size=12301) 2024-11-18T20:08:42,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960582622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:42,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960582622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:42,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960582632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:42,674 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/1a8df4db0a8e4b18b525b89a3b816504 2024-11-18T20:08:42,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/5f5d6f39e2de47aea6b9c82450190066 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f5d6f39e2de47aea6b9c82450190066 2024-11-18T20:08:42,683 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f5d6f39e2de47aea6b9c82450190066, entries=150, sequenceid=497, filesize=12.0 K 2024-11-18T20:08:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/7ac1a1de5fd34a298a4512c58c7cdbc3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7ac1a1de5fd34a298a4512c58c7cdbc3 2024-11-18T20:08:42,690 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7ac1a1de5fd34a298a4512c58c7cdbc3, entries=150, sequenceid=497, filesize=12.0 K 2024-11-18T20:08:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/1a8df4db0a8e4b18b525b89a3b816504 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1a8df4db0a8e4b18b525b89a3b816504 2024-11-18T20:08:42,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1a8df4db0a8e4b18b525b89a3b816504, entries=150, sequenceid=497, filesize=12.0 K 2024-11-18T20:08:42,703 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for b989f3762007617cfc5051d63359b50d in 1274ms, sequenceid=497, compaction requested=false 2024-11-18T20:08:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-18T20:08:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-18T20:08:42,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T20:08:42,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8900 sec 2024-11-18T20:08:42,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8940 sec 2024-11-18T20:08:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:08:42,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:42,963 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-18T20:08:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:08:42,964 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:42,965 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:42,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:08:43,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T20:08:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:08:43,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fb1a4c7731a74e89a7ae15fb1f7b6346 is 50, key is test_row_0/A:col10/1731960521484/Put/seqid=0 2024-11-18T20:08:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741942_1118 (size=12301) 2024-11-18T20:08:43,154 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fb1a4c7731a74e89a7ae15fb1f7b6346 2024-11-18T20:08:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0e92b01ceb874520a3a2bcf2d250bd07 is 50, key is test_row_0/B:col10/1731960521484/Put/seqid=0 2024-11-18T20:08:43,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741943_1119 (size=12301) 2024-11-18T20:08:43,175 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0e92b01ceb874520a3a2bcf2d250bd07 2024-11-18T20:08:43,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/e7a6cb8256c440c3af02608f3d64350d is 50, key is test_row_0/C:col10/1731960521484/Put/seqid=0 2024-11-18T20:08:43,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741944_1120 (size=12301) 2024-11-18T20:08:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:08:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:08:43,594 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/e7a6cb8256c440c3af02608f3d64350d 2024-11-18T20:08:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/fb1a4c7731a74e89a7ae15fb1f7b6346 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fb1a4c7731a74e89a7ae15fb1f7b6346 2024-11-18T20:08:43,604 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fb1a4c7731a74e89a7ae15fb1f7b6346, entries=150, sequenceid=519, filesize=12.0 K 2024-11-18T20:08:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/0e92b01ceb874520a3a2bcf2d250bd07 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0e92b01ceb874520a3a2bcf2d250bd07 2024-11-18T20:08:43,610 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0e92b01ceb874520a3a2bcf2d250bd07, entries=150, sequenceid=519, filesize=12.0 K 2024-11-18T20:08:43,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/e7a6cb8256c440c3af02608f3d64350d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/e7a6cb8256c440c3af02608f3d64350d 2024-11-18T20:08:43,616 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/e7a6cb8256c440c3af02608f3d64350d, entries=150, sequenceid=519, filesize=12.0 K 2024-11-18T20:08:43,617 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for b989f3762007617cfc5051d63359b50d in 500ms, sequenceid=519, compaction requested=true 2024-11-18T20:08:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-18T20:08:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-18T20:08:43,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T20:08:43,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 654 msec 2024-11-18T20:08:43,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 657 msec 2024-11-18T20:08:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:43,652 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:43,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:43,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/35b8c748cbc0439596511f17d2cab880 is 50, key is test_row_0/A:col10/1731960523651/Put/seqid=0 2024-11-18T20:08:43,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741945_1121 (size=14741) 2024-11-18T20:08:43,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960583709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960583709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960583710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960583712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,735 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-18T20:08:43,735 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:43,736 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-18T20:08:43,737 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:43,737 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-18T20:08:43,737 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:43,740 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-18T20:08:43,740 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,740 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,740 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,740 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:43,740 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,741 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:43,741 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:43,742 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:43,742 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:43,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:43,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960583822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960583822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960583822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960583823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960584032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960584032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960584033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960584033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/35b8c748cbc0439596511f17d2cab880 2024-11-18T20:08:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:08:44,103 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:44,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/fb2c9c516ef749829d9d9cd40b1fd47f is 50, key is test_row_0/B:col10/1731960523651/Put/seqid=0 2024-11-18T20:08:44,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741946_1122 (size=12301) 2024-11-18T20:08:44,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960584343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960584343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960584343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960584344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/fb2c9c516ef749829d9d9cd40b1fd47f 2024-11-18T20:08:44,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4775a142c09a451ab5617ba971784b1a is 50, key is test_row_0/C:col10/1731960523651/Put/seqid=0 2024-11-18T20:08:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741947_1123 (size=12301) 2024-11-18T20:08:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58078 deadline: 1731960584852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58084 deadline: 1731960584852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1731960584852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58096 deadline: 1731960584853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:08:44,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4775a142c09a451ab5617ba971784b1a 2024-11-18T20:08:44,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/35b8c748cbc0439596511f17d2cab880 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/35b8c748cbc0439596511f17d2cab880 2024-11-18T20:08:44,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/35b8c748cbc0439596511f17d2cab880, entries=200, sequenceid=531, filesize=14.4 K 2024-11-18T20:08:44,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/fb2c9c516ef749829d9d9cd40b1fd47f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/fb2c9c516ef749829d9d9cd40b1fd47f 2024-11-18T20:08:44,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/fb2c9c516ef749829d9d9cd40b1fd47f, entries=150, sequenceid=531, filesize=12.0 K 2024-11-18T20:08:44,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/4775a142c09a451ab5617ba971784b1a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4775a142c09a451ab5617ba971784b1a 2024-11-18T20:08:44,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4775a142c09a451ab5617ba971784b1a, entries=150, sequenceid=531, filesize=12.0 K 2024-11-18T20:08:44,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for b989f3762007617cfc5051d63359b50d in 1312ms, sequenceid=531, compaction requested=true 2024-11-18T20:08:44,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:44,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:44,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:44,964 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:44,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:44,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:44,965 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:44,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b989f3762007617cfc5051d63359b50d:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:44,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:44,966 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:44,966 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:44,966 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/A is initiating minor compaction (all files) 2024-11-18T20:08:44,966 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/B is initiating minor compaction (all files) 2024-11-18T20:08:44,966 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/A in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:44,966 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/B in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:44,966 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0b0bd0dd0f8a473b9f22933a5e54527a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7ac1a1de5fd34a298a4512c58c7cdbc3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0e92b01ceb874520a3a2bcf2d250bd07, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/fb2c9c516ef749829d9d9cd40b1fd47f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=49.1 K 2024-11-18T20:08:44,966 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ff8904fde6d540c88de3459552c1d367, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f5d6f39e2de47aea6b9c82450190066, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fb1a4c7731a74e89a7ae15fb1f7b6346, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/35b8c748cbc0439596511f17d2cab880] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=51.5 K 2024-11-18T20:08:44,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b0bd0dd0f8a473b9f22933a5e54527a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731960520171 2024-11-18T20:08:44,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff8904fde6d540c88de3459552c1d367, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731960520171 2024-11-18T20:08:44,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f5d6f39e2de47aea6b9c82450190066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731960520829 2024-11-18T20:08:44,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac1a1de5fd34a298a4512c58c7cdbc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731960520829 2024-11-18T20:08:44,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb1a4c7731a74e89a7ae15fb1f7b6346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731960521479 2024-11-18T20:08:44,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e92b01ceb874520a3a2bcf2d250bd07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731960521479 2024-11-18T20:08:44,968 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35b8c748cbc0439596511f17d2cab880, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731960523638 2024-11-18T20:08:44,968 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2c9c516ef749829d9d9cd40b1fd47f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731960523638 2024-11-18T20:08:44,978 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#B#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:44,978 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#A#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:44,979 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/19ff3936731248d79ceca77eb8afb416 is 50, key is test_row_0/B:col10/1731960523651/Put/seqid=0 2024-11-18T20:08:44,979 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c6208a67fb2f482d9ff452f013a60eaf is 50, key is test_row_0/A:col10/1731960523651/Put/seqid=0 2024-11-18T20:08:44,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741949_1125 (size=13561) 2024-11-18T20:08:44,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741948_1124 (size=13561) 2024-11-18T20:08:45,396 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/c6208a67fb2f482d9ff452f013a60eaf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c6208a67fb2f482d9ff452f013a60eaf 2024-11-18T20:08:45,397 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/19ff3936731248d79ceca77eb8afb416 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/19ff3936731248d79ceca77eb8afb416 2024-11-18T20:08:45,402 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/A of b989f3762007617cfc5051d63359b50d into c6208a67fb2f482d9ff452f013a60eaf(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:45,402 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/B of b989f3762007617cfc5051d63359b50d into 19ff3936731248d79ceca77eb8afb416(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:45,402 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:45,402 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:45,402 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/B, priority=12, startTime=1731960524964; duration=0sec 2024-11-18T20:08:45,402 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/A, priority=12, startTime=1731960524964; duration=0sec 2024-11-18T20:08:45,402 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:45,403 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:B 2024-11-18T20:08:45,403 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:45,403 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:A 2024-11-18T20:08:45,403 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:08:45,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:08:45,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): b989f3762007617cfc5051d63359b50d/C is initiating minor compaction (all files) 2024-11-18T20:08:45,404 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b989f3762007617cfc5051d63359b50d/C in TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:45,404 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4ee44e538b57440f8a9b34ceee597c8f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1a8df4db0a8e4b18b525b89a3b816504, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/e7a6cb8256c440c3af02608f3d64350d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4775a142c09a451ab5617ba971784b1a] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp, totalSize=49.1 K 2024-11-18T20:08:45,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee44e538b57440f8a9b34ceee597c8f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731960520171 2024-11-18T20:08:45,405 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a8df4db0a8e4b18b525b89a3b816504, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731960520829 2024-11-18T20:08:45,405 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a6cb8256c440c3af02608f3d64350d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731960521479 2024-11-18T20:08:45,406 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4775a142c09a451ab5617ba971784b1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731960523638 2024-11-18T20:08:45,415 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b989f3762007617cfc5051d63359b50d#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:45,416 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/cd484359f12c44ac9a07a41d8dc1a6e8 is 50, key is test_row_0/C:col10/1731960523651/Put/seqid=0 2024-11-18T20:08:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741950_1126 (size=13561) 2024-11-18T20:08:45,833 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/cd484359f12c44ac9a07a41d8dc1a6e8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/cd484359f12c44ac9a07a41d8dc1a6e8 2024-11-18T20:08:45,840 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b989f3762007617cfc5051d63359b50d/C of b989f3762007617cfc5051d63359b50d into cd484359f12c44ac9a07a41d8dc1a6e8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:45,840 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:45,840 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., storeName=b989f3762007617cfc5051d63359b50d/C, priority=12, startTime=1731960524965; duration=0sec 2024-11-18T20:08:45,840 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:45,840 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b989f3762007617cfc5051d63359b50d:C 2024-11-18T20:08:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:45,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:45,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:08:45,864 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-18T20:08:45,864 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:45,864 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,864 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:45,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:08:45,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81222f813c294c988c1124a67d1b2082 is 50, key is test_row_0/A:col10/1731960523709/Put/seqid=0 2024-11-18T20:08:45,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741951_1127 (size=12301) 2024-11-18T20:08:45,876 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-18T20:08:45,876 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:45,876 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,876 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-18T20:08:45,876 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-18T20:08:45,876 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:45,876 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,876 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:45,876 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:45,876 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,876 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,876 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:45,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:45,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:45,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81222f813c294c988c1124a67d1b2082 2024-11-18T20:08:45,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/23eed8cec1f74d3cbba0fec2fd4e35ac is 50, key is test_row_0/B:col10/1731960523709/Put/seqid=0 2024-11-18T20:08:45,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741952_1128 (size=12301) 2024-11-18T20:08:46,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/23eed8cec1f74d3cbba0fec2fd4e35ac 2024-11-18T20:08:46,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/361503be6666413daa0fa64684a7116e is 50, key is test_row_0/C:col10/1731960523709/Put/seqid=0 2024-11-18T20:08:46,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741953_1129 (size=12301) 2024-11-18T20:08:46,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/361503be6666413daa0fa64684a7116e 2024-11-18T20:08:46,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/81222f813c294c988c1124a67d1b2082 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81222f813c294c988c1124a67d1b2082 2024-11-18T20:08:46,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81222f813c294c988c1124a67d1b2082, entries=150, sequenceid=560, filesize=12.0 K 2024-11-18T20:08:46,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/23eed8cec1f74d3cbba0fec2fd4e35ac as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/23eed8cec1f74d3cbba0fec2fd4e35ac 2024-11-18T20:08:46,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/23eed8cec1f74d3cbba0fec2fd4e35ac, entries=150, sequenceid=560, filesize=12.0 K 2024-11-18T20:08:46,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/361503be6666413daa0fa64684a7116e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/361503be6666413daa0fa64684a7116e 2024-11-18T20:08:46,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/361503be6666413daa0fa64684a7116e, entries=150, sequenceid=560, filesize=12.0 K 2024-11-18T20:08:46,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=20.13 KB/20610 for b989f3762007617cfc5051d63359b50d in 875ms, sequenceid=560, compaction requested=false 2024-11-18T20:08:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b989f3762007617cfc5051d63359b50d: 2024-11-18T20:08:49,077 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-18T20:08:49,077 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:08:49,077 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:49,078 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:49,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:49,078 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 130 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:08:49,079 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3466 2024-11-18T20:08:49,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3564 2024-11-18T20:08:49,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:08:49,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1571 2024-11-18T20:08:49,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4713 rows 2024-11-18T20:08:49,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1578 2024-11-18T20:08:49,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4733 rows 2024-11-18T20:08:49,080 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:08:49,081 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:08:49,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:49,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:49,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:08:49,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:08:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:08:49,093 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960529093"}]},"ts":"1731960529093"} 2024-11-18T20:08:49,095 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:08:49,095 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:08:49,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:08:49,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, UNASSIGN}] 2024-11-18T20:08:49,103 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, UNASSIGN 2024-11-18T20:08:49,105 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b989f3762007617cfc5051d63359b50d, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:49,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, UNASSIGN because future has completed 2024-11-18T20:08:49,108 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:08:49,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:08:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:08:49,268 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:49,269 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:08:49,270 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing b989f3762007617cfc5051d63359b50d, disabling compactions & flushes 2024-11-18T20:08:49,270 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:49,270 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:49,270 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. after waiting 0 ms 2024-11-18T20:08:49,270 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:49,270 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing b989f3762007617cfc5051d63359b50d 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-18T20:08:49,276 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/bff642eb215646c6b0eee5718eb2f183 is 50, key is test_row_1/A:col10/1731960529073/Put/seqid=0 2024-11-18T20:08:49,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741954_1130 (size=9857) 2024-11-18T20:08:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:08:49,681 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/bff642eb215646c6b0eee5718eb2f183 2024-11-18T20:08:49,694 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/580c289fbf8f4d9b9b2961f1537b7f93 is 50, key is test_row_1/B:col10/1731960529073/Put/seqid=0 2024-11-18T20:08:49,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741955_1131 (size=9857) 2024-11-18T20:08:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:08:50,099 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/580c289fbf8f4d9b9b2961f1537b7f93 2024-11-18T20:08:50,111 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/21b2f7277ad149bf9828dd3a58a8bd3c is 50, key is test_row_1/C:col10/1731960529073/Put/seqid=0 2024-11-18T20:08:50,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741956_1132 (size=9857) 2024-11-18T20:08:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:08:50,518 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/21b2f7277ad149bf9828dd3a58a8bd3c 2024-11-18T20:08:50,524 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/A/bff642eb215646c6b0eee5718eb2f183 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bff642eb215646c6b0eee5718eb2f183 2024-11-18T20:08:50,529 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bff642eb215646c6b0eee5718eb2f183, entries=100, sequenceid=567, filesize=9.6 K 2024-11-18T20:08:50,531 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/B/580c289fbf8f4d9b9b2961f1537b7f93 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/580c289fbf8f4d9b9b2961f1537b7f93 2024-11-18T20:08:50,537 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/580c289fbf8f4d9b9b2961f1537b7f93, entries=100, sequenceid=567, filesize=9.6 K 2024-11-18T20:08:50,538 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/.tmp/C/21b2f7277ad149bf9828dd3a58a8bd3c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/21b2f7277ad149bf9828dd3a58a8bd3c 2024-11-18T20:08:50,544 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/21b2f7277ad149bf9828dd3a58a8bd3c, entries=100, sequenceid=567, filesize=9.6 K 2024-11-18T20:08:50,545 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b989f3762007617cfc5051d63359b50d in 1275ms, sequenceid=567, compaction requested=true 2024-11-18T20:08:50,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81da7a93e1714736b5df02123626d5cc, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/0e8b756a49184ac18e358ac898dafe9c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/800948c99f234ad6a4d566a66dfb168b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ade3b8211b56448fa1b428b040ca7422, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/df6b41aa51a44725909f6fe7cdbf4d46, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bddb56e2c5f34c51ac700ee3796c6261, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e06ef3de4be24624b018f6e2cb473947, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fba02b8959ab41e7bb9a854a9094f54e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81fdaaad17714da39791bdfa2ff7c4f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/aa4c80855dc3414c9d57fe06b4765851, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/92a44f53bcf34fbaa73475ddaaff2fa8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ae4fb4bb7f374639a05a1f539fee3a42, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/6916b77f3f8b46ccb1fe7c4dd44711b7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/857d16faa05a4fb48111f2ad167a1b2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f23ce75cf004bc68b2e726d23112e45, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b6fb4b04022a4964bd286292e5617052, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/01aaf077a11f48b5b4e2f50faa9f84f0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/073229826af94a6cbbfaad968717bd18, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ffb275be42d14576b21cdb1e3553d1e6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b522494cf8d24175a231375a554f0cdd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e587cdb8a3624c1fb371affe2e29f3c3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fbb823655c9444929e0c89952bbb00a3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/f4ed6eafffd7460998d91e6ac8d5f6de, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8f5eb178ac534d32b10a4ca2df705c32, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/2a9284166f204faa86f8d2e578829a6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e7a01981550a46918f0cc4d4bff50481, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/75daee86595c431099c4a1d848966ad8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8ccdd4a5d25b42cca5f876899cc39341, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/4bb566df49384f3caeffd9a791667fff, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c01c1bec744d4796b9b8882f4d6ee8c1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/d34f974192514051ad174d49b9b56f4b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/21a11fdac53a4e6393404dbd157dc99a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c12173e356b244ddbb98137a5a4b121b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ff8904fde6d540c88de3459552c1d367, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f5d6f39e2de47aea6b9c82450190066, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fb1a4c7731a74e89a7ae15fb1f7b6346, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/35b8c748cbc0439596511f17d2cab880] to archive 2024-11-18T20:08:50,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:08:50,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81da7a93e1714736b5df02123626d5cc to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81da7a93e1714736b5df02123626d5cc 2024-11-18T20:08:50,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/0e8b756a49184ac18e358ac898dafe9c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/0e8b756a49184ac18e358ac898dafe9c 2024-11-18T20:08:50,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/800948c99f234ad6a4d566a66dfb168b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/800948c99f234ad6a4d566a66dfb168b 2024-11-18T20:08:50,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ade3b8211b56448fa1b428b040ca7422 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ade3b8211b56448fa1b428b040ca7422 2024-11-18T20:08:50,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/df6b41aa51a44725909f6fe7cdbf4d46 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/df6b41aa51a44725909f6fe7cdbf4d46 2024-11-18T20:08:50,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bddb56e2c5f34c51ac700ee3796c6261 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bddb56e2c5f34c51ac700ee3796c6261 2024-11-18T20:08:50,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e06ef3de4be24624b018f6e2cb473947 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e06ef3de4be24624b018f6e2cb473947 2024-11-18T20:08:50,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fba02b8959ab41e7bb9a854a9094f54e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fba02b8959ab41e7bb9a854a9094f54e 2024-11-18T20:08:50,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81fdaaad17714da39791bdfa2ff7c4f8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81fdaaad17714da39791bdfa2ff7c4f8 2024-11-18T20:08:50,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/aa4c80855dc3414c9d57fe06b4765851 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/aa4c80855dc3414c9d57fe06b4765851 2024-11-18T20:08:50,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/92a44f53bcf34fbaa73475ddaaff2fa8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/92a44f53bcf34fbaa73475ddaaff2fa8 2024-11-18T20:08:50,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ae4fb4bb7f374639a05a1f539fee3a42 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ae4fb4bb7f374639a05a1f539fee3a42 2024-11-18T20:08:50,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/6916b77f3f8b46ccb1fe7c4dd44711b7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/6916b77f3f8b46ccb1fe7c4dd44711b7 2024-11-18T20:08:50,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/857d16faa05a4fb48111f2ad167a1b2c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/857d16faa05a4fb48111f2ad167a1b2c 2024-11-18T20:08:50,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f23ce75cf004bc68b2e726d23112e45 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f23ce75cf004bc68b2e726d23112e45 2024-11-18T20:08:50,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b6fb4b04022a4964bd286292e5617052 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b6fb4b04022a4964bd286292e5617052 2024-11-18T20:08:50,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/01aaf077a11f48b5b4e2f50faa9f84f0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/01aaf077a11f48b5b4e2f50faa9f84f0 2024-11-18T20:08:50,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/073229826af94a6cbbfaad968717bd18 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/073229826af94a6cbbfaad968717bd18 2024-11-18T20:08:50,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ffb275be42d14576b21cdb1e3553d1e6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ffb275be42d14576b21cdb1e3553d1e6 2024-11-18T20:08:50,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b522494cf8d24175a231375a554f0cdd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/b522494cf8d24175a231375a554f0cdd 2024-11-18T20:08:50,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e587cdb8a3624c1fb371affe2e29f3c3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e587cdb8a3624c1fb371affe2e29f3c3 2024-11-18T20:08:50,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fbb823655c9444929e0c89952bbb00a3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fbb823655c9444929e0c89952bbb00a3 2024-11-18T20:08:50,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/f4ed6eafffd7460998d91e6ac8d5f6de to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/f4ed6eafffd7460998d91e6ac8d5f6de 2024-11-18T20:08:50,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8f5eb178ac534d32b10a4ca2df705c32 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8f5eb178ac534d32b10a4ca2df705c32 2024-11-18T20:08:50,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/2a9284166f204faa86f8d2e578829a6f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/2a9284166f204faa86f8d2e578829a6f 2024-11-18T20:08:50,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e7a01981550a46918f0cc4d4bff50481 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/e7a01981550a46918f0cc4d4bff50481 2024-11-18T20:08:50,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/75daee86595c431099c4a1d848966ad8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/75daee86595c431099c4a1d848966ad8 2024-11-18T20:08:50,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8ccdd4a5d25b42cca5f876899cc39341 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/8ccdd4a5d25b42cca5f876899cc39341 2024-11-18T20:08:50,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/4bb566df49384f3caeffd9a791667fff to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/4bb566df49384f3caeffd9a791667fff 2024-11-18T20:08:50,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c01c1bec744d4796b9b8882f4d6ee8c1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c01c1bec744d4796b9b8882f4d6ee8c1 2024-11-18T20:08:50,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/d34f974192514051ad174d49b9b56f4b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/d34f974192514051ad174d49b9b56f4b 2024-11-18T20:08:50,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/21a11fdac53a4e6393404dbd157dc99a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/21a11fdac53a4e6393404dbd157dc99a 2024-11-18T20:08:50,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c12173e356b244ddbb98137a5a4b121b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c12173e356b244ddbb98137a5a4b121b 2024-11-18T20:08:50,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ff8904fde6d540c88de3459552c1d367 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/ff8904fde6d540c88de3459552c1d367 2024-11-18T20:08:50,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f5d6f39e2de47aea6b9c82450190066 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/5f5d6f39e2de47aea6b9c82450190066 2024-11-18T20:08:50,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fb1a4c7731a74e89a7ae15fb1f7b6346 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/fb1a4c7731a74e89a7ae15fb1f7b6346 2024-11-18T20:08:50,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/35b8c748cbc0439596511f17d2cab880 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/35b8c748cbc0439596511f17d2cab880 2024-11-18T20:08:50,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/02524f77fe1447bb910c3e3e8b5726e9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/56418585ff524065994e1e8741b7ec0e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e18416a9205f46149dba5b84fc636001, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/878027fe0e2e437994787efdacefeb2e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/b07d9761ec7440f08222b1b034496824, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/82d223610c204183986a82d901edf919, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/66628fff626a43a29cfabdc00350252d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/64e758bf4fbf4bb3aebfbc4eebf28f3e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/8f21ff1c5f2e4a389a8f13a392e08bd6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/c7e9cfe761474868849deb985a7998bb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/888c7ba913df488ca1cd95618ee64d3d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0ff3e353399d4501b2f62e07899dfc2e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0c93f5ba679c4082964df4a22332b2b6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1270e12a09d8418b9e22acb60ed0a704, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f034dd7e1fd9492b987997a58321440d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0d75316fd5904cc79a6a41e098058ad7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/a4da0f9258504fac9a795ccbbc9ec322, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2007b52bcad6495993288223c8b7421a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f42f129d1720405eb3a846748e9c7d29, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1d7d46f882a44c67bb775c7a02720bd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/599ea7f2228448e48f3754e19c3ec978, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/4b3df0425c1045ffb62bf109cbba4b6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/450e7fbedc5c45dfb4a388a31bf7033b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/75f8018abdb54325a2aa0af187eb3e30, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f78b7223162949ab9d2e91b1806cd73b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/54e074b7257146b98326ff8eddecee54, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7cf6c74683cd430f9564970a28e3fd1c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3ad9648dcb0e4ecfbaa9e580226b8ac2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f97191c8253b43f49f2c30ed6da1cb2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/16a8e3c0eeab4e27849e3235fb6e240c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e34aa8d4a782432c800c44f04115eee1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0b0bd0dd0f8a473b9f22933a5e54527a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2a44c75247af41d29bb54d30301d2b90, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7ac1a1de5fd34a298a4512c58c7cdbc3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0e92b01ceb874520a3a2bcf2d250bd07, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/fb2c9c516ef749829d9d9cd40b1fd47f] to archive 2024-11-18T20:08:50,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:08:50,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/02524f77fe1447bb910c3e3e8b5726e9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/02524f77fe1447bb910c3e3e8b5726e9 2024-11-18T20:08:50,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/56418585ff524065994e1e8741b7ec0e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/56418585ff524065994e1e8741b7ec0e 2024-11-18T20:08:50,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3e4cc6ccfd5e410f8b79d7b76d2a7a29 2024-11-18T20:08:50,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e18416a9205f46149dba5b84fc636001 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e18416a9205f46149dba5b84fc636001 2024-11-18T20:08:50,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/878027fe0e2e437994787efdacefeb2e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/878027fe0e2e437994787efdacefeb2e 2024-11-18T20:08:50,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/b07d9761ec7440f08222b1b034496824 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/b07d9761ec7440f08222b1b034496824 2024-11-18T20:08:50,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/82d223610c204183986a82d901edf919 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/82d223610c204183986a82d901edf919 2024-11-18T20:08:50,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/66628fff626a43a29cfabdc00350252d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/66628fff626a43a29cfabdc00350252d 2024-11-18T20:08:50,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/64e758bf4fbf4bb3aebfbc4eebf28f3e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/64e758bf4fbf4bb3aebfbc4eebf28f3e 2024-11-18T20:08:50,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/8f21ff1c5f2e4a389a8f13a392e08bd6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/8f21ff1c5f2e4a389a8f13a392e08bd6 2024-11-18T20:08:50,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/c7e9cfe761474868849deb985a7998bb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/c7e9cfe761474868849deb985a7998bb 2024-11-18T20:08:50,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/888c7ba913df488ca1cd95618ee64d3d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/888c7ba913df488ca1cd95618ee64d3d 2024-11-18T20:08:50,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0ff3e353399d4501b2f62e07899dfc2e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0ff3e353399d4501b2f62e07899dfc2e 2024-11-18T20:08:50,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0c93f5ba679c4082964df4a22332b2b6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0c93f5ba679c4082964df4a22332b2b6 2024-11-18T20:08:50,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1270e12a09d8418b9e22acb60ed0a704 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1270e12a09d8418b9e22acb60ed0a704 2024-11-18T20:08:50,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f034dd7e1fd9492b987997a58321440d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f034dd7e1fd9492b987997a58321440d 2024-11-18T20:08:50,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0d75316fd5904cc79a6a41e098058ad7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0d75316fd5904cc79a6a41e098058ad7 2024-11-18T20:08:50,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/a4da0f9258504fac9a795ccbbc9ec322 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/a4da0f9258504fac9a795ccbbc9ec322 2024-11-18T20:08:50,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2007b52bcad6495993288223c8b7421a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2007b52bcad6495993288223c8b7421a 2024-11-18T20:08:50,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f42f129d1720405eb3a846748e9c7d29 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f42f129d1720405eb3a846748e9c7d29 2024-11-18T20:08:50,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1d7d46f882a44c67bb775c7a02720bd3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/1d7d46f882a44c67bb775c7a02720bd3 2024-11-18T20:08:50,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/599ea7f2228448e48f3754e19c3ec978 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/599ea7f2228448e48f3754e19c3ec978 2024-11-18T20:08:50,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/4b3df0425c1045ffb62bf109cbba4b6f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/4b3df0425c1045ffb62bf109cbba4b6f 2024-11-18T20:08:50,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/450e7fbedc5c45dfb4a388a31bf7033b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/450e7fbedc5c45dfb4a388a31bf7033b 2024-11-18T20:08:50,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/75f8018abdb54325a2aa0af187eb3e30 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/75f8018abdb54325a2aa0af187eb3e30 2024-11-18T20:08:50,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f78b7223162949ab9d2e91b1806cd73b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f78b7223162949ab9d2e91b1806cd73b 2024-11-18T20:08:50,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/54e074b7257146b98326ff8eddecee54 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/54e074b7257146b98326ff8eddecee54 2024-11-18T20:08:50,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7cf6c74683cd430f9564970a28e3fd1c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7cf6c74683cd430f9564970a28e3fd1c 2024-11-18T20:08:50,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3ad9648dcb0e4ecfbaa9e580226b8ac2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/3ad9648dcb0e4ecfbaa9e580226b8ac2 2024-11-18T20:08:50,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f97191c8253b43f49f2c30ed6da1cb2c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/f97191c8253b43f49f2c30ed6da1cb2c 2024-11-18T20:08:50,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/16a8e3c0eeab4e27849e3235fb6e240c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/16a8e3c0eeab4e27849e3235fb6e240c 2024-11-18T20:08:50,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e34aa8d4a782432c800c44f04115eee1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/e34aa8d4a782432c800c44f04115eee1 2024-11-18T20:08:50,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0b0bd0dd0f8a473b9f22933a5e54527a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0b0bd0dd0f8a473b9f22933a5e54527a 2024-11-18T20:08:50,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2a44c75247af41d29bb54d30301d2b90 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/2a44c75247af41d29bb54d30301d2b90 2024-11-18T20:08:50,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7ac1a1de5fd34a298a4512c58c7cdbc3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/7ac1a1de5fd34a298a4512c58c7cdbc3 2024-11-18T20:08:50,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0e92b01ceb874520a3a2bcf2d250bd07 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/0e92b01ceb874520a3a2bcf2d250bd07 2024-11-18T20:08:50,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/fb2c9c516ef749829d9d9cd40b1fd47f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/fb2c9c516ef749829d9d9cd40b1fd47f 2024-11-18T20:08:50,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/8bc08c165e414b71885c043c523adfa4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d769af64f65040d1b8059f33cb988804, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1f085217667c4e96b8347bb53ea22b05, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fb597b3d7b7d4117ac71807bb5d9bbcd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/516d98c335fb468f96b9bf86349efb19, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e2d472209334937883579a52502c179, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/95adccf8e77643aab90ca613bb0ff395, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/58256f68fadd453baef9819b1cdc0c3f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/140eea8321cd432180151d9f2a133fb4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e1da25de104489999d9eb297079a974, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/f63faef58c7c486397324b65810accfa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/c31a8c9cf62a483b9c1b38dfb71b5dd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/7288633d79924a1994850606b5ab4825, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/76a4685436894c3e8eeb5b688009d689, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e709ddb6bda43698061553d65726621, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/bd207320389e418489a1639794881927, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/225bac3a6b8e4d4194748f5b0c226873, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b2e4dd4aa848466182a5f4a51bfd0d39, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/26b020f0593a44d3afbb1f50b65527a6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/3176a5eecef84b638d75a34ac21b2127, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/edd4e42c2e534d57a376541aa9b1c3d8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/5b8ff1d79f694d6d940f02f503fefd3b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b41469fc011e4a99bcd1ded0406493e7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/ac09bbb182cb45ca96a1b6f0cd4cb856, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fc128d79e6c84b56a130bb4d3700e061, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/19b483cd5d6f48468d329d8d5815f966, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/72b73fa9083f4d91b8e9b7dbc0596879, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4c1409af7bf64c07a234b17cc503ea9a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d294af8fb21847629829822bed427850, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/66c3086ee12541648322f183c090abac, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4deeb2488cba4507bc0e5848bb4ab394, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b6f5d3b042da494a8b72219f9ae6d3de, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4ee44e538b57440f8a9b34ceee597c8f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/0eaba45f19b248249aede2deef759446, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1a8df4db0a8e4b18b525b89a3b816504, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/e7a6cb8256c440c3af02608f3d64350d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4775a142c09a451ab5617ba971784b1a] to archive 2024-11-18T20:08:50,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:08:50,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/8bc08c165e414b71885c043c523adfa4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/8bc08c165e414b71885c043c523adfa4 2024-11-18T20:08:50,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d769af64f65040d1b8059f33cb988804 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d769af64f65040d1b8059f33cb988804 2024-11-18T20:08:50,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1f085217667c4e96b8347bb53ea22b05 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1f085217667c4e96b8347bb53ea22b05 2024-11-18T20:08:50,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fb597b3d7b7d4117ac71807bb5d9bbcd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fb597b3d7b7d4117ac71807bb5d9bbcd 2024-11-18T20:08:50,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/516d98c335fb468f96b9bf86349efb19 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/516d98c335fb468f96b9bf86349efb19 2024-11-18T20:08:50,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e2d472209334937883579a52502c179 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e2d472209334937883579a52502c179 2024-11-18T20:08:50,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/95adccf8e77643aab90ca613bb0ff395 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/95adccf8e77643aab90ca613bb0ff395 2024-11-18T20:08:50,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/58256f68fadd453baef9819b1cdc0c3f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/58256f68fadd453baef9819b1cdc0c3f 2024-11-18T20:08:50,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/140eea8321cd432180151d9f2a133fb4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/140eea8321cd432180151d9f2a133fb4 2024-11-18T20:08:50,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e1da25de104489999d9eb297079a974 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e1da25de104489999d9eb297079a974 2024-11-18T20:08:50,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/f63faef58c7c486397324b65810accfa to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/f63faef58c7c486397324b65810accfa 2024-11-18T20:08:50,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/c31a8c9cf62a483b9c1b38dfb71b5dd3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/c31a8c9cf62a483b9c1b38dfb71b5dd3 2024-11-18T20:08:50,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/7288633d79924a1994850606b5ab4825 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/7288633d79924a1994850606b5ab4825 2024-11-18T20:08:50,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/76a4685436894c3e8eeb5b688009d689 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/76a4685436894c3e8eeb5b688009d689 2024-11-18T20:08:50,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e709ddb6bda43698061553d65726621 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/9e709ddb6bda43698061553d65726621 2024-11-18T20:08:50,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/bd207320389e418489a1639794881927 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/bd207320389e418489a1639794881927 2024-11-18T20:08:50,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/225bac3a6b8e4d4194748f5b0c226873 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/225bac3a6b8e4d4194748f5b0c226873 2024-11-18T20:08:50,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b2e4dd4aa848466182a5f4a51bfd0d39 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b2e4dd4aa848466182a5f4a51bfd0d39 2024-11-18T20:08:50,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/26b020f0593a44d3afbb1f50b65527a6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/26b020f0593a44d3afbb1f50b65527a6 2024-11-18T20:08:50,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/3176a5eecef84b638d75a34ac21b2127 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/3176a5eecef84b638d75a34ac21b2127 2024-11-18T20:08:50,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/edd4e42c2e534d57a376541aa9b1c3d8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/edd4e42c2e534d57a376541aa9b1c3d8 2024-11-18T20:08:50,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/5b8ff1d79f694d6d940f02f503fefd3b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/5b8ff1d79f694d6d940f02f503fefd3b 2024-11-18T20:08:50,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b41469fc011e4a99bcd1ded0406493e7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b41469fc011e4a99bcd1ded0406493e7 2024-11-18T20:08:50,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/ac09bbb182cb45ca96a1b6f0cd4cb856 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/ac09bbb182cb45ca96a1b6f0cd4cb856 2024-11-18T20:08:50,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fc128d79e6c84b56a130bb4d3700e061 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/fc128d79e6c84b56a130bb4d3700e061 2024-11-18T20:08:50,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/19b483cd5d6f48468d329d8d5815f966 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/19b483cd5d6f48468d329d8d5815f966 2024-11-18T20:08:50,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/72b73fa9083f4d91b8e9b7dbc0596879 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/72b73fa9083f4d91b8e9b7dbc0596879 2024-11-18T20:08:50,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4c1409af7bf64c07a234b17cc503ea9a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4c1409af7bf64c07a234b17cc503ea9a 2024-11-18T20:08:50,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d294af8fb21847629829822bed427850 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/d294af8fb21847629829822bed427850 2024-11-18T20:08:50,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/66c3086ee12541648322f183c090abac to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/66c3086ee12541648322f183c090abac 2024-11-18T20:08:50,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4deeb2488cba4507bc0e5848bb4ab394 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4deeb2488cba4507bc0e5848bb4ab394 2024-11-18T20:08:50,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b6f5d3b042da494a8b72219f9ae6d3de to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/b6f5d3b042da494a8b72219f9ae6d3de 2024-11-18T20:08:50,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4ee44e538b57440f8a9b34ceee597c8f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4ee44e538b57440f8a9b34ceee597c8f 2024-11-18T20:08:50,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/0eaba45f19b248249aede2deef759446 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/0eaba45f19b248249aede2deef759446 2024-11-18T20:08:50,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1a8df4db0a8e4b18b525b89a3b816504 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/1a8df4db0a8e4b18b525b89a3b816504 2024-11-18T20:08:50,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/e7a6cb8256c440c3af02608f3d64350d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/e7a6cb8256c440c3af02608f3d64350d 2024-11-18T20:08:50,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4775a142c09a451ab5617ba971784b1a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/4775a142c09a451ab5617ba971784b1a 2024-11-18T20:08:50,710 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/recovered.edits/570.seqid, newMaxSeqId=570, maxSeqId=1 2024-11-18T20:08:50,710 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. 2024-11-18T20:08:50,710 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for b989f3762007617cfc5051d63359b50d: Waiting for close lock at 1731960529270Running coprocessor pre-close hooks at 1731960529270Disabling compacts and flushes for region at 1731960529270Disabling writes for close at 1731960529270Obtaining lock to block concurrent updates at 1731960529270Preparing flush snapshotting stores in b989f3762007617cfc5051d63359b50d at 1731960529270Finished memstore snapshotting TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731960529271 (+1 ms)Flushing stores of TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d. at 1731960529272 (+1 ms)Flushing b989f3762007617cfc5051d63359b50d/A: creating writer at 1731960529272Flushing b989f3762007617cfc5051d63359b50d/A: appending metadata at 1731960529276 (+4 ms)Flushing b989f3762007617cfc5051d63359b50d/A: closing flushed file at 1731960529276Flushing b989f3762007617cfc5051d63359b50d/B: creating writer at 1731960529690 (+414 ms)Flushing b989f3762007617cfc5051d63359b50d/B: appending metadata at 1731960529693 (+3 ms)Flushing b989f3762007617cfc5051d63359b50d/B: closing flushed file at 1731960529693Flushing b989f3762007617cfc5051d63359b50d/C: creating writer at 1731960530106 (+413 ms)Flushing b989f3762007617cfc5051d63359b50d/C: appending metadata at 1731960530110 (+4 ms)Flushing b989f3762007617cfc5051d63359b50d/C: closing flushed file at 1731960530110Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63e0473c: reopening flushed file at 1731960530523 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f324fdc: reopening flushed file at 1731960530530 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15af062e: reopening flushed file at 1731960530537 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b989f3762007617cfc5051d63359b50d in 1275ms, sequenceid=567, compaction requested=true at 1731960530545 (+8 ms)Writing region close event to WAL at 1731960530706 (+161 ms)Running coprocessor post-close hooks at 1731960530710 (+4 ms)Closed at 1731960530710 2024-11-18T20:08:50,713 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:50,713 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b989f3762007617cfc5051d63359b50d, regionState=CLOSED 2024-11-18T20:08:50,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:08:50,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-18T20:08:50,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure b989f3762007617cfc5051d63359b50d, server=d8f95bb18b58,42447,1731960497538 in 1.6080 sec 2024-11-18T20:08:50,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-18T20:08:50,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b989f3762007617cfc5051d63359b50d, UNASSIGN in 1.6140 sec 2024-11-18T20:08:50,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T20:08:50,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6230 sec 2024-11-18T20:08:50,723 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960530722"}]},"ts":"1731960530722"} 2024-11-18T20:08:50,724 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:08:50,724 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:08:50,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6370 sec 2024-11-18T20:08:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:08:51,243 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:51,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:08:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,258 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:08:51,260 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,265 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:51,270 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/recovered.edits] 2024-11-18T20:08:51,273 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81222f813c294c988c1124a67d1b2082 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/81222f813c294c988c1124a67d1b2082 2024-11-18T20:08:51,275 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bff642eb215646c6b0eee5718eb2f183 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/bff642eb215646c6b0eee5718eb2f183 2024-11-18T20:08:51,276 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c6208a67fb2f482d9ff452f013a60eaf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/A/c6208a67fb2f482d9ff452f013a60eaf 2024-11-18T20:08:51,278 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/19ff3936731248d79ceca77eb8afb416 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/19ff3936731248d79ceca77eb8afb416 2024-11-18T20:08:51,280 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/23eed8cec1f74d3cbba0fec2fd4e35ac to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/23eed8cec1f74d3cbba0fec2fd4e35ac 2024-11-18T20:08:51,281 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/580c289fbf8f4d9b9b2961f1537b7f93 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/B/580c289fbf8f4d9b9b2961f1537b7f93 2024-11-18T20:08:51,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/21b2f7277ad149bf9828dd3a58a8bd3c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/21b2f7277ad149bf9828dd3a58a8bd3c 2024-11-18T20:08:51,285 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/361503be6666413daa0fa64684a7116e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/361503be6666413daa0fa64684a7116e 2024-11-18T20:08:51,286 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/cd484359f12c44ac9a07a41d8dc1a6e8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/C/cd484359f12c44ac9a07a41d8dc1a6e8 2024-11-18T20:08:51,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/recovered.edits/570.seqid to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d/recovered.edits/570.seqid 2024-11-18T20:08:51,289 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/b989f3762007617cfc5051d63359b50d 2024-11-18T20:08:51,289 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:08:51,294 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T20:08:51,304 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:08:51,335 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:08:51,336 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,337 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:08:51,337 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960531337"}]},"ts":"9223372036854775807"} 2024-11-18T20:08:51,341 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:08:51,341 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b989f3762007617cfc5051d63359b50d, NAME => 'TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:08:51,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:08:51,341 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960531341"}]},"ts":"9223372036854775807"} 2024-11-18T20:08:51,344 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:08:51,345 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-18T20:08:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T20:08:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:08:51,363 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:51,378 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1190570139_22 at /127.0.0.1:43976 [Waiting for operation #329] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d8f95bb18b58:42447-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1249326010_22 at /127.0.0.1:48078 [Waiting for operation #78] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=555 (was 441) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1338 (was 1737) 2024-11-18T20:08:51,389 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=555, ProcessCount=11, AvailableMemoryMB=1338 2024-11-18T20:08:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:08:51,391 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:08:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:51,393 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:08:51,394 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:51,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-18T20:08:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T20:08:51,395 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:08:51,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741957_1133 (size=959) 2024-11-18T20:08:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T20:08:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T20:08:51,808 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:08:51,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741958_1134 (size=53) 2024-11-18T20:08:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T20:08:52,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:52,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 43f9b782225aaf40f26cb68658ea7b20, disabling compactions & flushes 2024-11-18T20:08:52,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. after waiting 0 ms 2024-11-18T20:08:52,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 43f9b782225aaf40f26cb68658ea7b20: Waiting for close lock at 1731960532220Disabling compacts and flushes for region at 1731960532220Disabling writes for close at 1731960532221 (+1 ms)Writing region close event to WAL at 1731960532221Closed at 1731960532221 2024-11-18T20:08:52,223 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:08:52,224 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960532223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960532223"}]},"ts":"1731960532223"} 2024-11-18T20:08:52,227 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:08:52,228 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:08:52,229 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960532228"}]},"ts":"1731960532228"} 2024-11-18T20:08:52,231 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:08:52,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, ASSIGN}] 2024-11-18T20:08:52,233 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, ASSIGN 2024-11-18T20:08:52,233 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, ASSIGN; state=OFFLINE, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=false 2024-11-18T20:08:52,385 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:52,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, ASSIGN because future has completed 2024-11-18T20:08:52,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:08:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T20:08:52,551 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,551 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:08:52,551 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,551 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:52,552 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,552 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,554 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,556 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f9b782225aaf40f26cb68658ea7b20 columnFamilyName A 2024-11-18T20:08:52,556 DEBUG [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:52,557 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(327): Store=43f9b782225aaf40f26cb68658ea7b20/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:52,557 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,559 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f9b782225aaf40f26cb68658ea7b20 columnFamilyName B 2024-11-18T20:08:52,559 DEBUG [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:52,559 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(327): Store=43f9b782225aaf40f26cb68658ea7b20/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:52,559 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,561 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f9b782225aaf40f26cb68658ea7b20 columnFamilyName C 2024-11-18T20:08:52,561 DEBUG [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:52,561 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(327): Store=43f9b782225aaf40f26cb68658ea7b20/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:52,561 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,562 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,562 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,563 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,563 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,564 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:08:52,565 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,567 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:08:52,567 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 43f9b782225aaf40f26cb68658ea7b20; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73548522, jitterRate=0.09595838189125061}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:08:52,568 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:52,568 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 43f9b782225aaf40f26cb68658ea7b20: Running coprocessor pre-open hook at 1731960532552Writing region info on filesystem at 1731960532552Initializing all the Stores at 1731960532553 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960532553Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960532553Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960532553Cleaning up temporary data from old regions at 1731960532563 (+10 ms)Running coprocessor post-open hooks at 1731960532568 (+5 ms)Region opened successfully at 1731960532568 2024-11-18T20:08:52,569 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., pid=32, masterSystemTime=1731960532545 2024-11-18T20:08:52,571 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,571 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:52,571 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:52,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:08:52,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-18T20:08:52,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 in 182 msec 2024-11-18T20:08:52,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-18T20:08:52,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, ASSIGN in 344 msec 2024-11-18T20:08:52,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:08:52,578 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960532578"}]},"ts":"1731960532578"} 2024-11-18T20:08:52,579 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:08:52,580 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:08:52,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-11-18T20:08:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-18T20:08:53,542 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:53,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:53,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:53,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:53,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168a0083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:53,553 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f289009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:53,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:53,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:53,560 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:08:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:08:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@281d24f1 2024-11-18T20:08:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:08:53,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:08:53,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:08:53,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:08:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:08:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741959_1135 (size=995) 2024-11-18T20:08:53,989 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-18T20:08:53,989 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-18T20:08:53,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:08:54,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, REOPEN/MOVE}] 2024-11-18T20:08:54,004 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, REOPEN/MOVE 2024-11-18T20:08:54,005 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, REOPEN/MOVE because future has completed 2024-11-18T20:08:54,007 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:08:54,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:08:54,161 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,161 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:08:54,162 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 43f9b782225aaf40f26cb68658ea7b20, disabling compactions & flushes 2024-11-18T20:08:54,162 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,162 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,162 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. after waiting 0 ms 2024-11-18T20:08:54,162 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,171 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:08:54,172 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,172 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 43f9b782225aaf40f26cb68658ea7b20: Waiting for close lock at 1731960534162Running coprocessor pre-close hooks at 1731960534162Disabling compacts and flushes for region at 1731960534162Disabling writes for close at 1731960534162Writing region close event to WAL at 1731960534163 (+1 ms)Running coprocessor post-close hooks at 1731960534172 (+9 ms)Closed at 1731960534172 2024-11-18T20:08:54,172 WARN [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 43f9b782225aaf40f26cb68658ea7b20 to self. 2024-11-18T20:08:54,176 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,177 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=CLOSED 2024-11-18T20:08:54,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:08:54,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-18T20:08:54,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 in 174 msec 2024-11-18T20:08:54,186 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, REOPEN/MOVE; state=CLOSED, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=true 2024-11-18T20:08:54,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, REOPEN/MOVE because future has completed 2024-11-18T20:08:54,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:08:54,501 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,501 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:08:54,501 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,501 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:08:54,502 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,502 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,504 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,509 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f9b782225aaf40f26cb68658ea7b20 columnFamilyName A 2024-11-18T20:08:54,511 DEBUG [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:54,511 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(327): Store=43f9b782225aaf40f26cb68658ea7b20/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:54,512 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,512 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f9b782225aaf40f26cb68658ea7b20 columnFamilyName B 2024-11-18T20:08:54,512 DEBUG [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:54,513 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(327): Store=43f9b782225aaf40f26cb68658ea7b20/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:54,513 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,513 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43f9b782225aaf40f26cb68658ea7b20 columnFamilyName C 2024-11-18T20:08:54,513 DEBUG [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:54,514 INFO [StoreOpener-43f9b782225aaf40f26cb68658ea7b20-1 {}] regionserver.HStore(327): Store=43f9b782225aaf40f26cb68658ea7b20/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:08:54,514 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,514 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,515 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,516 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,516 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,517 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:08:54,518 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,519 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 43f9b782225aaf40f26cb68658ea7b20; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72918365, jitterRate=0.08656831085681915}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:08:54,519 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,521 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 43f9b782225aaf40f26cb68658ea7b20: Running coprocessor pre-open hook at 1731960534502Writing region info on filesystem at 1731960534502Initializing all the Stores at 1731960534503 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960534503Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960534503Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960534503Cleaning up temporary data from old regions at 1731960534516 (+13 ms)Running coprocessor post-open hooks at 1731960534519 (+3 ms)Region opened successfully at 1731960534521 (+2 ms) 2024-11-18T20:08:54,521 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., pid=37, masterSystemTime=1731960534495 2024-11-18T20:08:54,523 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,524 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,524 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=OPEN, openSeqNum=5, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:08:54,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-18T20:08:54,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 in 184 msec 2024-11-18T20:08:54,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-18T20:08:54,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, REOPEN/MOVE in 525 msec 2024-11-18T20:08:54,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-18T20:08:54,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-11-18T20:08:54,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 966 msec 2024-11-18T20:08:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-18T20:08:54,539 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:54,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d72c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b87891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,545 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2149b6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,547 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@484a7761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498357ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21912204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6136cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ff40dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2576a4a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62385257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,563 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550a689f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,565 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20459354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,567 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237ef6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,568 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3eeec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,569 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,570 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d643a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,571 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5784fa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:08:54,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:08:54,573 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:08:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:08:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:08:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bfe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:08:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:08:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:08:54,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:08:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e7ba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:08:54,575 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,575 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,575 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,575 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,575 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,576 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,576 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,577 DEBUG [Thread-677 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,577 DEBUG [Thread-680 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,578 DEBUG [Thread-686 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,578 DEBUG [Thread-680 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,578 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,578 DEBUG [Thread-686 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:54,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,579 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:08:54,580 DEBUG [Thread-677 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-18T20:08:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,583 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:08:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:08:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,585 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:54,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:08:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:08:54,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:54,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a5f65a0b433b45b09c86ab8dcaa44eca_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960534604/Put/seqid=0 2024-11-18T20:08:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960594653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960594654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960594659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960594661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960594667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741960_1136 (size=12154) 2024-11-18T20:08:54,680 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a5f65a0b433b45b09c86ab8dcaa44eca_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a5f65a0b433b45b09c86ab8dcaa44eca_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:54,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/d64be8a7a7ef47ffb04cbe652440c5ad, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:54,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/d64be8a7a7ef47ffb04cbe652440c5ad is 175, key is test_row_0/A:col10/1731960534604/Put/seqid=0 2024-11-18T20:08:54,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741961_1137 (size=30955) 2024-11-18T20:08:54,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/d64be8a7a7ef47ffb04cbe652440c5ad 2024-11-18T20:08:54,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:54,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960594773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960594773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960594782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960594783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960594784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a5121bcf11c0446fa09c7bbf93d75447 is 50, key is test_row_0/B:col10/1731960534604/Put/seqid=0 2024-11-18T20:08:54,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741962_1138 (size=12001) 2024-11-18T20:08:54,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a5121bcf11c0446fa09c7bbf93d75447 2024-11-18T20:08:54,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/c63cc2f2588a45e5b0e379b99aa038f8 is 50, key is test_row_0/C:col10/1731960534604/Put/seqid=0 2024-11-18T20:08:54,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741963_1139 (size=12001) 2024-11-18T20:08:54,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:54,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960594983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960594983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960594992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960594994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960594993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:55,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/c63cc2f2588a45e5b0e379b99aa038f8 2024-11-18T20:08:55,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/d64be8a7a7ef47ffb04cbe652440c5ad as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad 2024-11-18T20:08:55,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad, entries=150, sequenceid=16, filesize=30.2 K 2024-11-18T20:08:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a5121bcf11c0446fa09c7bbf93d75447 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5121bcf11c0446fa09c7bbf93d75447 2024-11-18T20:08:55,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5121bcf11c0446fa09c7bbf93d75447, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:08:55,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/c63cc2f2588a45e5b0e379b99aa038f8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/c63cc2f2588a45e5b0e379b99aa038f8 2024-11-18T20:08:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960595292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960595292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/c63cc2f2588a45e5b0e379b99aa038f8, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:08:55,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 43f9b782225aaf40f26cb68658ea7b20 in 695ms, sequenceid=16, compaction requested=false 2024-11-18T20:08:55,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:55,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:08:55,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ee12f3ff853349f7b52a0d86df719029_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960534653/Put/seqid=0 2024-11-18T20:08:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960595329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960595331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741964_1140 (size=17034) 2024-11-18T20:08:55,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960595340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,353 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ee12f3ff853349f7b52a0d86df719029_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee12f3ff853349f7b52a0d86df719029_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:55,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a0bb593437174566be4151e118a48f15, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:55,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a0bb593437174566be4151e118a48f15 is 175, key is test_row_0/A:col10/1731960534653/Put/seqid=0 2024-11-18T20:08:55,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741965_1141 (size=48139) 2024-11-18T20:08:55,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960595442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960595452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960595452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960595652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960595662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960595662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:55,771 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a0bb593437174566be4151e118a48f15 2024-11-18T20:08:55,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/090b4edb6c6c44dcb7062d16537d2b31 is 50, key is test_row_0/B:col10/1731960534653/Put/seqid=0 2024-11-18T20:08:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960595802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960595802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:55,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741966_1142 (size=12001) 2024-11-18T20:08:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/090b4edb6c6c44dcb7062d16537d2b31 2024-11-18T20:08:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/29cf8325cfd94330aff38c440cffb644 is 50, key is test_row_0/C:col10/1731960534653/Put/seqid=0 2024-11-18T20:08:55,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741967_1143 (size=12001) 2024-11-18T20:08:55,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/29cf8325cfd94330aff38c440cffb644 2024-11-18T20:08:55,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a0bb593437174566be4151e118a48f15 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15 2024-11-18T20:08:55,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15, entries=250, sequenceid=42, filesize=47.0 K 2024-11-18T20:08:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/090b4edb6c6c44dcb7062d16537d2b31 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/090b4edb6c6c44dcb7062d16537d2b31 2024-11-18T20:08:55,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/090b4edb6c6c44dcb7062d16537d2b31, entries=150, sequenceid=42, filesize=11.7 K 2024-11-18T20:08:55,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/29cf8325cfd94330aff38c440cffb644 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/29cf8325cfd94330aff38c440cffb644 2024-11-18T20:08:55,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/29cf8325cfd94330aff38c440cffb644, entries=150, sequenceid=42, filesize=11.7 K 2024-11-18T20:08:55,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 43f9b782225aaf40f26cb68658ea7b20 in 582ms, sequenceid=42, compaction requested=false 2024-11-18T20:08:55,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:55,966 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:55,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:55,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:55,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:55,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118700c374ce00e462ead17d8bb5d19aa48_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741968_1144 (size=12154) 2024-11-18T20:08:56,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:56,011 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118700c374ce00e462ead17d8bb5d19aa48_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118700c374ce00e462ead17d8bb5d19aa48_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:56,015 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/5483cbfef96b49b0a00ab565a84871cb, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:56,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/5483cbfef96b49b0a00ab565a84871cb is 175, key is test_row_0/A:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:56,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741969_1145 (size=30955) 2024-11-18T20:08:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960596040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,048 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/5483cbfef96b49b0a00ab565a84871cb 2024-11-18T20:08:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960596047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960596048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/099d9817c680459d9dd994be3b833bd7 is 50, key is test_row_0/B:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741970_1146 (size=12001) 2024-11-18T20:08:56,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960596152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960596161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960596162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,293 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:08:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960596362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960596372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960596373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:56,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:56,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/099d9817c680459d9dd994be3b833bd7 2024-11-18T20:08:56,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/578e77c5505a4ed3b5c04653e0034b96 is 50, key is test_row_0/C:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:56,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741971_1147 (size=12001) 2024-11-18T20:08:56,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960596672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960596682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960596692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:56,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960596812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,813 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960596822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:56,825 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:56,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:56,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/578e77c5505a4ed3b5c04653e0034b96 2024-11-18T20:08:56,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/5483cbfef96b49b0a00ab565a84871cb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb 2024-11-18T20:08:56,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb, entries=150, sequenceid=53, filesize=30.2 K 2024-11-18T20:08:56,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/099d9817c680459d9dd994be3b833bd7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/099d9817c680459d9dd994be3b833bd7 2024-11-18T20:08:56,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/099d9817c680459d9dd994be3b833bd7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-18T20:08:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/578e77c5505a4ed3b5c04653e0034b96 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/578e77c5505a4ed3b5c04653e0034b96 2024-11-18T20:08:56,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/578e77c5505a4ed3b5c04653e0034b96, entries=150, sequenceid=53, filesize=11.7 K 2024-11-18T20:08:56,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 43f9b782225aaf40f26cb68658ea7b20 in 1012ms, sequenceid=53, compaction requested=true 2024-11-18T20:08:56,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:56,979 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:56,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:56,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:56,979 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:56,982 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:56,982 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:56,982 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:08:56,982 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:08:56,982 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,982 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,982 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5121bcf11c0446fa09c7bbf93d75447, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/090b4edb6c6c44dcb7062d16537d2b31, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/099d9817c680459d9dd994be3b833bd7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.2 K 2024-11-18T20:08:56,982 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=107.5 K 2024-11-18T20:08:56,982 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:56,982 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb] 2024-11-18T20:08:56,983 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64be8a7a7ef47ffb04cbe652440c5ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960534588 2024-11-18T20:08:56,983 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a5121bcf11c0446fa09c7bbf93d75447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960534588 2024-11-18T20:08:56,984 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0bb593437174566be4151e118a48f15, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731960534649 2024-11-18T20:08:56,984 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 090b4edb6c6c44dcb7062d16537d2b31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731960534653 2024-11-18T20:08:56,984 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5483cbfef96b49b0a00ab565a84871cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960535318 2024-11-18T20:08:56,984 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 099d9817c680459d9dd994be3b833bd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960535318 2024-11-18T20:08:56,999 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:57,003 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:57,004 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a4a68020051e44a9bfecf34e3f314985 is 50, key is test_row_0/B:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:57,004 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411185d889c191f2e4715a6e8ee537aff5824_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:57,008 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411185d889c191f2e4715a6e8ee537aff5824_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:57,009 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185d889c191f2e4715a6e8ee537aff5824_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:57,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741972_1148 (size=12104) 2024-11-18T20:08:57,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741973_1149 (size=4469) 2024-11-18T20:08:57,049 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#129 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:57,050 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/5f662595dbc04fb6a027ddd3516f7319 is 175, key is test_row_0/A:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:57,051 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a4a68020051e44a9bfecf34e3f314985 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a4a68020051e44a9bfecf34e3f314985 2024-11-18T20:08:57,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T20:08:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:57,057 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:08:57,064 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into a4a68020051e44a9bfecf34e3f314985(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:57,064 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:57,065 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=13, startTime=1731960536979; duration=0sec 2024-11-18T20:08:57,065 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:57,065 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:08:57,065 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:57,066 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:57,066 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:08:57,066 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:57,066 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/c63cc2f2588a45e5b0e379b99aa038f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/29cf8325cfd94330aff38c440cffb644, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/578e77c5505a4ed3b5c04653e0034b96] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.2 K 2024-11-18T20:08:57,067 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting c63cc2f2588a45e5b0e379b99aa038f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960534588 2024-11-18T20:08:57,067 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 29cf8325cfd94330aff38c440cffb644, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731960534653 2024-11-18T20:08:57,067 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 578e77c5505a4ed3b5c04653e0034b96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960535318 2024-11-18T20:08:57,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741974_1150 (size=31058) 2024-11-18T20:08:57,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180e7212161d1b4101a5bbde7d7e867d00_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960536044/Put/seqid=0 2024-11-18T20:08:57,086 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:57,086 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/5ea0bb46a3374fdcad8b3ca229d89db0 is 50, key is test_row_0/C:col10/1731960535331/Put/seqid=0 2024-11-18T20:08:57,089 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/5f662595dbc04fb6a027ddd3516f7319 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5f662595dbc04fb6a027ddd3516f7319 2024-11-18T20:08:57,097 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into 5f662595dbc04fb6a027ddd3516f7319(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:57,097 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:57,097 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=13, startTime=1731960536979; duration=0sec 2024-11-18T20:08:57,097 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:57,097 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:08:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741976_1152 (size=12104) 2024-11-18T20:08:57,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741975_1151 (size=12154) 2024-11-18T20:08:57,134 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/5ea0bb46a3374fdcad8b3ca229d89db0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/5ea0bb46a3374fdcad8b3ca229d89db0 2024-11-18T20:08:57,145 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into 5ea0bb46a3374fdcad8b3ca229d89db0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:57,145 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:57,145 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=13, startTime=1731960536980; duration=0sec 2024-11-18T20:08:57,146 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:57,146 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:08:57,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:57,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960597198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960597200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960597203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960597312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960597312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960597523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960597525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,526 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180e7212161d1b4101a5bbde7d7e867d00_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180e7212161d1b4101a5bbde7d7e867d00_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/e122017e7aa940d6a12ee6d277457ff5, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/e122017e7aa940d6a12ee6d277457ff5 is 175, key is test_row_0/A:col10/1731960536044/Put/seqid=0 2024-11-18T20:08:57,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741977_1153 (size=30955) 2024-11-18T20:08:57,538 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/e122017e7aa940d6a12ee6d277457ff5 2024-11-18T20:08:57,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/b8f697be5f714b9da4bba6e86c3880f9 is 50, key is test_row_0/B:col10/1731960536044/Put/seqid=0 2024-11-18T20:08:57,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741978_1154 (size=12001) 2024-11-18T20:08:57,581 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/b8f697be5f714b9da4bba6e86c3880f9 2024-11-18T20:08:57,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/0097d7689f0d48dc8f6ad23f2e94b83a is 50, key is test_row_0/C:col10/1731960536044/Put/seqid=0 2024-11-18T20:08:57,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741979_1155 (size=12001) 2024-11-18T20:08:57,611 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/0097d7689f0d48dc8f6ad23f2e94b83a 2024-11-18T20:08:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/e122017e7aa940d6a12ee6d277457ff5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5 2024-11-18T20:08:57,628 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5, entries=150, sequenceid=79, filesize=30.2 K 2024-11-18T20:08:57,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/b8f697be5f714b9da4bba6e86c3880f9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/b8f697be5f714b9da4bba6e86c3880f9 2024-11-18T20:08:57,636 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/b8f697be5f714b9da4bba6e86c3880f9, entries=150, sequenceid=79, filesize=11.7 K 2024-11-18T20:08:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/0097d7689f0d48dc8f6ad23f2e94b83a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/0097d7689f0d48dc8f6ad23f2e94b83a 2024-11-18T20:08:57,646 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/0097d7689f0d48dc8f6ad23f2e94b83a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-18T20:08:57,648 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 43f9b782225aaf40f26cb68658ea7b20 in 591ms, sequenceid=79, compaction requested=false 2024-11-18T20:08:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-18T20:08:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-18T20:08:57,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-18T20:08:57,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0640 sec 2024-11-18T20:08:57,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.0740 sec 2024-11-18T20:08:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:57,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:08:57,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:08:57,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111877ec4c5eeeff44d1b20d8be52a61cb22_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:57,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741980_1156 (size=14594) 2024-11-18T20:08:57,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960597933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:57,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960597937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960598041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960598052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960598212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,214 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960598252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,259 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:58,262 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111877ec4c5eeeff44d1b20d8be52a61cb22_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111877ec4c5eeeff44d1b20d8be52a61cb22_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:58,263 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/66a3a1576d074d7da2f1b0ffe94f2d30, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:58,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/66a3a1576d074d7da2f1b0ffe94f2d30 is 175, key is test_row_0/A:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:58,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960598262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741981_1157 (size=39549) 2024-11-18T20:08:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960598562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960598572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/66a3a1576d074d7da2f1b0ffe94f2d30 2024-11-18T20:08:58,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a03d5f2cbb8046168d486b99b99e4133 is 50, key is test_row_0/B:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:58,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741982_1158 (size=12001) 2024-11-18T20:08:58,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a03d5f2cbb8046168d486b99b99e4133 2024-11-18T20:08:58,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/3c42d44722ce49c7811d2da81cd6e428 is 50, key is test_row_0/C:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:58,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741983_1159 (size=12001) 2024-11-18T20:08:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T20:08:58,762 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:08:58,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:08:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-18T20:08:58,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:08:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:08:58,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:08:58,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:08:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960598841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,842 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960598852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:58,855 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:08:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:08:58,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:08:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:58,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:58,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:59,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:08:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:08:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:59,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960599072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:08:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:08:59,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:08:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960599083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:08:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:08:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:08:59,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/3c42d44722ce49c7811d2da81cd6e428 2024-11-18T20:08:59,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/66a3a1576d074d7da2f1b0ffe94f2d30 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30 2024-11-18T20:08:59,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30, entries=200, sequenceid=93, filesize=38.6 K 2024-11-18T20:08:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a03d5f2cbb8046168d486b99b99e4133 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a03d5f2cbb8046168d486b99b99e4133 2024-11-18T20:08:59,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a03d5f2cbb8046168d486b99b99e4133, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:08:59,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/3c42d44722ce49c7811d2da81cd6e428 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3c42d44722ce49c7811d2da81cd6e428 2024-11-18T20:08:59,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3c42d44722ce49c7811d2da81cd6e428, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:08:59,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 43f9b782225aaf40f26cb68658ea7b20 in 1293ms, sequenceid=93, compaction requested=true 2024-11-18T20:08:59,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:59,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:08:59,127 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:59,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:59,127 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:08:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:08:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:59,128 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:59,129 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:08:59,129 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:59,129 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:08:59,129 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,129 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,129 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5f662595dbc04fb6a027ddd3516f7319, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=99.2 K 2024-11-18T20:08:59,129 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a4a68020051e44a9bfecf34e3f314985, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/b8f697be5f714b9da4bba6e86c3880f9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a03d5f2cbb8046168d486b99b99e4133] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.3 K 2024-11-18T20:08:59,129 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,129 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5f662595dbc04fb6a027ddd3516f7319, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30] 2024-11-18T20:08:59,130 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f662595dbc04fb6a027ddd3516f7319, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960535318 2024-11-18T20:08:59,130 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a68020051e44a9bfecf34e3f314985, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960535318 2024-11-18T20:08:59,130 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f697be5f714b9da4bba6e86c3880f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731960536035 2024-11-18T20:08:59,131 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e122017e7aa940d6a12ee6d277457ff5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731960536035 2024-11-18T20:08:59,131 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a03d5f2cbb8046168d486b99b99e4133, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960537192 2024-11-18T20:08:59,132 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a3a1576d074d7da2f1b0ffe94f2d30, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960537192 2024-11-18T20:08:59,145 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:59,146 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e679ad48ce314b4f97b8f7ad3bb1b63d is 50, key is test_row_0/B:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:59,146 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:59,160 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118a3b10920c59d4b0c9e22137ad5eff80a_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:59,163 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118a3b10920c59d4b0c9e22137ad5eff80a_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:59,165 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a3b10920c59d4b0c9e22137ad5eff80a_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:59,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741984_1160 (size=12207) 2024-11-18T20:08:59,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741985_1161 (size=4469) 2024-11-18T20:08:59,198 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#139 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:59,198 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/d6e25af9fc84472ab7161e723244fb24 is 175, key is test_row_0/A:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741986_1162 (size=31161) 2024-11-18T20:08:59,219 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/d6e25af9fc84472ab7161e723244fb24 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d6e25af9fc84472ab7161e723244fb24 2024-11-18T20:08:59,227 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into d6e25af9fc84472ab7161e723244fb24(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:59,227 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:59,227 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=13, startTime=1731960539127; duration=0sec 2024-11-18T20:08:59,227 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:08:59,227 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:08:59,227 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:08:59,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:08:59,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,228 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:08:59,230 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:08:59,230 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:08:59,230 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:08:59,230 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/5ea0bb46a3374fdcad8b3ca229d89db0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/0097d7689f0d48dc8f6ad23f2e94b83a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3c42d44722ce49c7811d2da81cd6e428] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.3 K 2024-11-18T20:08:59,234 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea0bb46a3374fdcad8b3ca229d89db0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960535318 2024-11-18T20:08:59,237 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0097d7689f0d48dc8f6ad23f2e94b83a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731960536035 2024-11-18T20:08:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118087c678f266c4b68afb7e5d7842f7c26_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960537935/Put/seqid=0 2024-11-18T20:08:59,238 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c42d44722ce49c7811d2da81cd6e428, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960537192 2024-11-18T20:08:59,262 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:08:59,263 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/e7d9dba8ddc64d99bc135093ee229160 is 50, key is test_row_0/C:col10/1731960537192/Put/seqid=0 2024-11-18T20:08:59,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741988_1164 (size=12207) 2024-11-18T20:08:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741987_1163 (size=12154) 2024-11-18T20:08:59,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:08:59,298 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118087c678f266c4b68afb7e5d7842f7c26_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118087c678f266c4b68afb7e5d7842f7c26_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:08:59,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/eb699d1b43854f53a85cc754d4780abb, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:08:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/eb699d1b43854f53a85cc754d4780abb is 175, key is test_row_0/A:col10/1731960537935/Put/seqid=0 2024-11-18T20:08:59,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741989_1165 (size=30955) 2024-11-18T20:08:59,330 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/eb699d1b43854f53a85cc754d4780abb 2024-11-18T20:08:59,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/5934750f6e814ae6aac5d4fdd130c336 is 50, key is test_row_0/B:col10/1731960537935/Put/seqid=0 2024-11-18T20:08:59,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741990_1166 (size=12001) 2024-11-18T20:08:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:08:59,598 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e679ad48ce314b4f97b8f7ad3bb1b63d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e679ad48ce314b4f97b8f7ad3bb1b63d 2024-11-18T20:08:59,603 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into e679ad48ce314b4f97b8f7ad3bb1b63d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:59,603 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:59,603 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=13, startTime=1731960539127; duration=0sec 2024-11-18T20:08:59,603 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:59,604 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:08:59,681 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/e7d9dba8ddc64d99bc135093ee229160 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/e7d9dba8ddc64d99bc135093ee229160 2024-11-18T20:08:59,687 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into e7d9dba8ddc64d99bc135093ee229160(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:08:59,687 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:08:59,687 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=13, startTime=1731960539128; duration=0sec 2024-11-18T20:08:59,687 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:08:59,687 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:08:59,784 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/5934750f6e814ae6aac5d4fdd130c336 2024-11-18T20:08:59,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/4e4d92a0f502445d98fb551b5ea2b9d9 is 50, key is test_row_0/C:col10/1731960537935/Put/seqid=0 2024-11-18T20:08:59,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741991_1167 (size=12001) 2024-11-18T20:08:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:09:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:00,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960600099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960600100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,199 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/4e4d92a0f502445d98fb551b5ea2b9d9 2024-11-18T20:09:00,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960600202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/eb699d1b43854f53a85cc754d4780abb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb 2024-11-18T20:09:00,210 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb, entries=150, sequenceid=119, filesize=30.2 K 2024-11-18T20:09:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/5934750f6e814ae6aac5d4fdd130c336 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5934750f6e814ae6aac5d4fdd130c336 2024-11-18T20:09:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960600212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,216 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5934750f6e814ae6aac5d4fdd130c336, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T20:09:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/4e4d92a0f502445d98fb551b5ea2b9d9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4e4d92a0f502445d98fb551b5ea2b9d9 2024-11-18T20:09:00,222 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4e4d92a0f502445d98fb551b5ea2b9d9, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T20:09:00,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 43f9b782225aaf40f26cb68658ea7b20 in 1014ms, sequenceid=119, compaction requested=false 2024-11-18T20:09:00,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:00,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:00,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-18T20:09:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-18T20:09:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:00,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:00,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:09:00,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-18T20:09:00,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-18T20:09:00,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.4840 sec 2024-11-18T20:09:00,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866fbf618b30f446c93c0d31a4f920334_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:00,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741992_1168 (size=14794) 2024-11-18T20:09:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960600312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960600412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960600421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960600422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960600632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:00,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866fbf618b30f446c93c0d31a4f920334_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866fbf618b30f446c93c0d31a4f920334_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:00,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/2e06aec5a72948dbb97abcb47a7f2e4c, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:00,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/2e06aec5a72948dbb97abcb47a7f2e4c is 175, key is test_row_0/A:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:00,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741993_1169 (size=39749) 2024-11-18T20:09:00,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960600722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960600732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:09:00,912 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:00,913 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-18T20:09:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:09:00,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:00,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:00,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:00,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960600942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:09:01,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:09:01,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:01,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/2e06aec5a72948dbb97abcb47a7f2e4c 2024-11-18T20:09:01,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/3c58a281d69d4e56ad8c0fc9b77bfccf is 50, key is test_row_0/B:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:01,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741994_1170 (size=12151) 2024-11-18T20:09:01,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/3c58a281d69d4e56ad8c0fc9b77bfccf 2024-11-18T20:09:01,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/d3e06a8e1f0244628dcc026c5ae1db7e is 50, key is test_row_0/C:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741995_1171 (size=12151) 2024-11-18T20:09:01,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:09:01,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:01,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:09:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960601235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960601250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:01,269 INFO [master/d8f95bb18b58:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T20:09:01,269 INFO [master/d8f95bb18b58:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T20:09:01,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:09:01,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:01,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960601452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:01,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/d3e06a8e1f0244628dcc026c5ae1db7e 2024-11-18T20:09:01,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/2e06aec5a72948dbb97abcb47a7f2e4c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c 2024-11-18T20:09:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:09:01,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:09:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c, entries=200, sequenceid=134, filesize=38.8 K 2024-11-18T20:09:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:01,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/3c58a281d69d4e56ad8c0fc9b77bfccf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/3c58a281d69d4e56ad8c0fc9b77bfccf 2024-11-18T20:09:01,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/3c58a281d69d4e56ad8c0fc9b77bfccf, entries=150, sequenceid=134, filesize=11.9 K 2024-11-18T20:09:01,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/d3e06a8e1f0244628dcc026c5ae1db7e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/d3e06a8e1f0244628dcc026c5ae1db7e 2024-11-18T20:09:01,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/d3e06a8e1f0244628dcc026c5ae1db7e, entries=150, sequenceid=134, filesize=11.9 K 2024-11-18T20:09:01,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 43f9b782225aaf40f26cb68658ea7b20 in 1315ms, sequenceid=134, compaction requested=true 2024-11-18T20:09:01,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:01,560 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:01,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:01,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:01,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:01,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:01,561 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:01,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:01,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:01,562 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:01,562 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:09:01,562 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,562 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d6e25af9fc84472ab7161e723244fb24, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=99.5 K 2024-11-18T20:09:01,562 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,562 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d6e25af9fc84472ab7161e723244fb24, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c] 2024-11-18T20:09:01,563 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:01,563 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6e25af9fc84472ab7161e723244fb24, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960537192 2024-11-18T20:09:01,563 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:09:01,563 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,563 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e679ad48ce314b4f97b8f7ad3bb1b63d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5934750f6e814ae6aac5d4fdd130c336, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/3c58a281d69d4e56ad8c0fc9b77bfccf] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.5 K 2024-11-18T20:09:01,563 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb699d1b43854f53a85cc754d4780abb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731960537925 2024-11-18T20:09:01,563 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e679ad48ce314b4f97b8f7ad3bb1b63d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960537192 2024-11-18T20:09:01,563 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e06aec5a72948dbb97abcb47a7f2e4c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960540097 2024-11-18T20:09:01,564 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5934750f6e814ae6aac5d4fdd130c336, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731960537925 2024-11-18T20:09:01,564 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c58a281d69d4e56ad8c0fc9b77bfccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960540097 2024-11-18T20:09:01,572 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:01,577 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:01,578 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/6e2a4dd896da447c83de339f96dc80a5 is 50, key is test_row_0/B:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:01,580 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411182f2e49cf79fd4fe19daad321b60178cb_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:01,583 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411182f2e49cf79fd4fe19daad321b60178cb_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:01,583 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182f2e49cf79fd4fe19daad321b60178cb_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:01,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741997_1173 (size=4469) 2024-11-18T20:09:01,591 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#147 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:01,592 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/fb76783522dd4cf5be70302d94f193c0 is 175, key is test_row_0/A:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:01,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741996_1172 (size=12459) 2024-11-18T20:09:01,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741998_1174 (size=31413) 2024-11-18T20:09:01,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:09:01,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:01,697 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:09:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111839b772ea028e406689bfa4ef5c6a79e5_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960540310/Put/seqid=0 2024-11-18T20:09:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741999_1175 (size=12304) 2024-11-18T20:09:01,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:01,727 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111839b772ea028e406689bfa4ef5c6a79e5_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111839b772ea028e406689bfa4ef5c6a79e5_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:01,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a6c927d7c23f4985a6e7f13314356a7d, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a6c927d7c23f4985a6e7f13314356a7d is 175, key is test_row_0/A:col10/1731960540310/Put/seqid=0 2024-11-18T20:09:01,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742000_1176 (size=31105) 2024-11-18T20:09:02,000 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/6e2a4dd896da447c83de339f96dc80a5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/6e2a4dd896da447c83de339f96dc80a5 2024-11-18T20:09:02,005 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into 6e2a4dd896da447c83de339f96dc80a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:02,005 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:02,005 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=13, startTime=1731960541561; duration=0sec 2024-11-18T20:09:02,005 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:02,005 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:09:02,005 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:02,009 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:02,009 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:09:02,009 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:02,009 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/e7d9dba8ddc64d99bc135093ee229160, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4e4d92a0f502445d98fb551b5ea2b9d9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/d3e06a8e1f0244628dcc026c5ae1db7e] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.5 K 2024-11-18T20:09:02,011 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d9dba8ddc64d99bc135093ee229160, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960537192 2024-11-18T20:09:02,012 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e4d92a0f502445d98fb551b5ea2b9d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731960537925 2024-11-18T20:09:02,012 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e06a8e1f0244628dcc026c5ae1db7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960540097 2024-11-18T20:09:02,014 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/fb76783522dd4cf5be70302d94f193c0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fb76783522dd4cf5be70302d94f193c0 2024-11-18T20:09:02,021 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into fb76783522dd4cf5be70302d94f193c0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:02,021 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:02,021 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=13, startTime=1731960541560; duration=0sec 2024-11-18T20:09:02,021 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:02,021 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:09:02,034 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:02,035 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/990a914f9e054861be6003bba2a6e75f is 50, key is test_row_0/C:col10/1731960540244/Put/seqid=0 2024-11-18T20:09:02,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742001_1177 (size=12459) 2024-11-18T20:09:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:09:02,058 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/990a914f9e054861be6003bba2a6e75f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/990a914f9e054861be6003bba2a6e75f 2024-11-18T20:09:02,063 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into 990a914f9e054861be6003bba2a6e75f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:02,067 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:02,067 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=13, startTime=1731960541561; duration=0sec 2024-11-18T20:09:02,067 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:02,067 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:09:02,152 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a6c927d7c23f4985a6e7f13314356a7d 2024-11-18T20:09:02,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/bf77496086294a2f82b752b417835c18 is 50, key is test_row_0/B:col10/1731960540310/Put/seqid=0 2024-11-18T20:09:02,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742002_1178 (size=12151) 2024-11-18T20:09:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:02,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:02,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960602268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960602272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960602381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960602381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960602472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,478 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,580 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/bf77496086294a2f82b752b417835c18 2024-11-18T20:09:02,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/ab9e73912c0c44d7bdae5f96506d9366 is 50, key is test_row_0/C:col10/1731960540310/Put/seqid=0 2024-11-18T20:09:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960602592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742003_1179 (size=12151) 2024-11-18T20:09:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960602592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,596 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/ab9e73912c0c44d7bdae5f96506d9366 2024-11-18T20:09:02,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/a6c927d7c23f4985a6e7f13314356a7d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d 2024-11-18T20:09:02,609 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d, entries=150, sequenceid=157, filesize=30.4 K 2024-11-18T20:09:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/bf77496086294a2f82b752b417835c18 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bf77496086294a2f82b752b417835c18 2024-11-18T20:09:02,614 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bf77496086294a2f82b752b417835c18, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:09:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/ab9e73912c0c44d7bdae5f96506d9366 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ab9e73912c0c44d7bdae5f96506d9366 2024-11-18T20:09:02,624 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ab9e73912c0c44d7bdae5f96506d9366, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:09:02,625 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 43f9b782225aaf40f26cb68658ea7b20 in 928ms, sequenceid=157, compaction requested=false 2024-11-18T20:09:02,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:02,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:02,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-18T20:09:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-18T20:09:02,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T20:09:02,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-11-18T20:09:02,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7150 sec 2024-11-18T20:09:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:02,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:09:02,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bf332ba768fb4849b9d0b5d0c83efcfa_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:02,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742004_1180 (size=14794) 2024-11-18T20:09:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960602908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960602913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960602915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960602916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960603022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960603022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960603022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960603031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:09:03,062 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:03,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-18T20:09:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:03,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:03,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:03,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:03,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960603232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960603232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960603232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960603242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,288 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:03,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bf332ba768fb4849b9d0b5d0c83efcfa_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bf332ba768fb4849b9d0b5d0c83efcfa_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:03,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/ca8dd678bf7c432c93a51f187099f486, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:03,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/ca8dd678bf7c432c93a51f187099f486 is 175, key is test_row_0/A:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:03,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742005_1181 (size=39749) 2024-11-18T20:09:03,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:03,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960603542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960603542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960603542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960603552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:03,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:03,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/ca8dd678bf7c432c93a51f187099f486 2024-11-18T20:09:03,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/405a68480ea14cdf99b06ee09db54a93 is 50, key is test_row_0/B:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:03,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742006_1182 (size=12151) 2024-11-18T20:09:03,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/405a68480ea14cdf99b06ee09db54a93 2024-11-18T20:09:03,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/7d1258d9b0b04e58b2c618f78144901f is 50, key is test_row_0/C:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:03,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742007_1183 (size=12151) 2024-11-18T20:09:03,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:03,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:03,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:03,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:04,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960604052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960604052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960604062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960604062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:04,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/7d1258d9b0b04e58b2c618f78144901f 2024-11-18T20:09:04,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/ca8dd678bf7c432c93a51f187099f486 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486 2024-11-18T20:09:04,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486, entries=200, sequenceid=174, filesize=38.8 K 2024-11-18T20:09:04,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/405a68480ea14cdf99b06ee09db54a93 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/405a68480ea14cdf99b06ee09db54a93 2024-11-18T20:09:04,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/405a68480ea14cdf99b06ee09db54a93, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:09:04,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/7d1258d9b0b04e58b2c618f78144901f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7d1258d9b0b04e58b2c618f78144901f 2024-11-18T20:09:04,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7d1258d9b0b04e58b2c618f78144901f, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:09:04,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 43f9b782225aaf40f26cb68658ea7b20 in 1283ms, sequenceid=174, compaction requested=true 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:04,158 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:04,158 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:04,159 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:04,159 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:04,159 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:09:04,159 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:09:04,159 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,159 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,159 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/6e2a4dd896da447c83de339f96dc80a5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bf77496086294a2f82b752b417835c18, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/405a68480ea14cdf99b06ee09db54a93] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.9 K 2024-11-18T20:09:04,159 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fb76783522dd4cf5be70302d94f193c0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=99.9 K 2024-11-18T20:09:04,159 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,159 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fb76783522dd4cf5be70302d94f193c0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486] 2024-11-18T20:09:04,162 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2a4dd896da447c83de339f96dc80a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960540097 2024-11-18T20:09:04,162 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb76783522dd4cf5be70302d94f193c0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960540097 2024-11-18T20:09:04,162 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c927d7c23f4985a6e7f13314356a7d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960540281 2024-11-18T20:09:04,162 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting bf77496086294a2f82b752b417835c18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960540281 2024-11-18T20:09:04,163 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8dd678bf7c432c93a51f187099f486, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960542263 2024-11-18T20:09:04,163 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 405a68480ea14cdf99b06ee09db54a93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960542267 2024-11-18T20:09:04,169 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:04,170 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:04,171 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/5e7317ac4daa4c559d82df02b823bcc4 is 50, key is test_row_0/B:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:04,178 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118c7057f80352d4f829ef8b865656807bb_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:04,180 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118c7057f80352d4f829ef8b865656807bb_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:04,181 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c7057f80352d4f829ef8b865656807bb_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742008_1184 (size=12561) 2024-11-18T20:09:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742009_1185 (size=4469) 2024-11-18T20:09:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:04,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:09:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,289 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:09:04,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b7d4f68d8f8142aba2ce83061ecf1a96_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960542904/Put/seqid=0 2024-11-18T20:09:04,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742010_1186 (size=12304) 2024-11-18T20:09:04,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:04,313 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b7d4f68d8f8142aba2ce83061ecf1a96_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b7d4f68d8f8142aba2ce83061ecf1a96_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/fd49d9970b0f48489082496f0b3a0d30, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/fd49d9970b0f48489082496f0b3a0d30 is 175, key is test_row_0/A:col10/1731960542904/Put/seqid=0 2024-11-18T20:09:04,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742011_1187 (size=31105) 2024-11-18T20:09:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:04,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960604534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:04,587 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:04,588 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/3ba09d0e9d7441098196c41445a8ce16 is 175, key is test_row_0/A:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:04,590 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/5e7317ac4daa4c559d82df02b823bcc4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5e7317ac4daa4c559d82df02b823bcc4 2024-11-18T20:09:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742012_1188 (size=31515) 2024-11-18T20:09:04,596 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into 5e7317ac4daa4c559d82df02b823bcc4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:04,597 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:04,597 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=13, startTime=1731960544158; duration=0sec 2024-11-18T20:09:04,597 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:04,597 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:09:04,597 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:04,598 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:04,598 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:09:04,598 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:04,598 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/990a914f9e054861be6003bba2a6e75f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ab9e73912c0c44d7bdae5f96506d9366, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7d1258d9b0b04e58b2c618f78144901f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=35.9 K 2024-11-18T20:09:04,598 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 990a914f9e054861be6003bba2a6e75f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960540097 2024-11-18T20:09:04,599 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9e73912c0c44d7bdae5f96506d9366, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960540281 2024-11-18T20:09:04,600 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d1258d9b0b04e58b2c618f78144901f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960542267 2024-11-18T20:09:04,600 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/3ba09d0e9d7441098196c41445a8ce16 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/3ba09d0e9d7441098196c41445a8ce16 2024-11-18T20:09:04,606 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into 3ba09d0e9d7441098196c41445a8ce16(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:04,606 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:04,606 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=13, startTime=1731960544158; duration=0sec 2024-11-18T20:09:04,606 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:04,606 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:09:04,608 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:04,608 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab is 50, key is test_row_0/C:col10/1731960542267/Put/seqid=0 2024-11-18T20:09:04,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742013_1189 (size=12561) 2024-11-18T20:09:04,628 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab 2024-11-18T20:09:04,633 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into 4c2f8e3627e743bd8c9b5ac7ed1d7cab(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:04,633 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:04,633 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=13, startTime=1731960544158; duration=0sec 2024-11-18T20:09:04,633 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:04,633 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:09:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960604642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:04,719 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/fd49d9970b0f48489082496f0b3a0d30 2024-11-18T20:09:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/2c1f87379b34414a9bc81bc915718b5b is 50, key is test_row_0/B:col10/1731960542904/Put/seqid=0 2024-11-18T20:09:04,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742014_1190 (size=12151) 2024-11-18T20:09:04,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960604852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960605062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960605062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,063 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,063 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960605072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960605072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,074 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,074 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,132 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/2c1f87379b34414a9bc81bc915718b5b 2024-11-18T20:09:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/9fecc4cd8dc04867bbe4caa1243ce673 is 50, key is test_row_0/C:col10/1731960542904/Put/seqid=0 2024-11-18T20:09:05,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742015_1191 (size=12151) 2024-11-18T20:09:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960605162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:05,543 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/9fecc4cd8dc04867bbe4caa1243ce673 2024-11-18T20:09:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/fd49d9970b0f48489082496f0b3a0d30 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30 2024-11-18T20:09:05,554 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30, entries=150, sequenceid=196, filesize=30.4 K 2024-11-18T20:09:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/2c1f87379b34414a9bc81bc915718b5b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/2c1f87379b34414a9bc81bc915718b5b 2024-11-18T20:09:05,560 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/2c1f87379b34414a9bc81bc915718b5b, entries=150, sequenceid=196, filesize=11.9 K 2024-11-18T20:09:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/9fecc4cd8dc04867bbe4caa1243ce673 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/9fecc4cd8dc04867bbe4caa1243ce673 2024-11-18T20:09:05,566 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/9fecc4cd8dc04867bbe4caa1243ce673, entries=150, sequenceid=196, filesize=11.9 K 2024-11-18T20:09:05,567 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 43f9b782225aaf40f26cb68658ea7b20 in 1278ms, sequenceid=196, compaction requested=false 2024-11-18T20:09:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-18T20:09:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-18T20:09:05,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T20:09:05,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5050 sec 2024-11-18T20:09:05,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.5090 sec 2024-11-18T20:09:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:05,674 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:05,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:09:05,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bb51ae4bb822455c8eba423c02363c5c_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:05,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742016_1192 (size=14794) 2024-11-18T20:09:05,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960605752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:05,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960605862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:05,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:05,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:06,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960606071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:06,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:06,091 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bb51ae4bb822455c8eba423c02363c5c_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bb51ae4bb822455c8eba423c02363c5c_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:06,093 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/007f3ce258e0403cb4be82edb35b58fb, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:06,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/007f3ce258e0403cb4be82edb35b58fb is 175, key is test_row_0/A:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:06,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742017_1193 (size=39749) 2024-11-18T20:09:06,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/007f3ce258e0403cb4be82edb35b58fb 2024-11-18T20:09:06,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/44f97433c4fa468c912348b94e2a3012 is 50, key is test_row_0/B:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:06,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742018_1194 (size=12151) 2024-11-18T20:09:06,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960606382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:06,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/44f97433c4fa468c912348b94e2a3012 2024-11-18T20:09:06,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/a0561827d33c426c970588678a86bd1a is 50, key is test_row_0/C:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:06,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742019_1195 (size=12151) 2024-11-18T20:09:06,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960606891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:06,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:06,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:06,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:06,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/a0561827d33c426c970588678a86bd1a 2024-11-18T20:09:06,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/007f3ce258e0403cb4be82edb35b58fb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb 2024-11-18T20:09:06,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb, entries=200, sequenceid=214, filesize=38.8 K 2024-11-18T20:09:06,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/44f97433c4fa468c912348b94e2a3012 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/44f97433c4fa468c912348b94e2a3012 2024-11-18T20:09:06,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/44f97433c4fa468c912348b94e2a3012, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:09:06,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/a0561827d33c426c970588678a86bd1a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a0561827d33c426c970588678a86bd1a 2024-11-18T20:09:06,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a0561827d33c426c970588678a86bd1a, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:09:06,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 43f9b782225aaf40f26cb68658ea7b20 in 1291ms, sequenceid=214, compaction requested=true 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:06,965 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:06,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:09:06,965 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:06,966 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:06,966 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:09:06,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:06,967 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:06,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:09:06,967 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:06,967 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/3ba09d0e9d7441098196c41445a8ce16, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=100.0 K 2024-11-18T20:09:06,967 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5e7317ac4daa4c559d82df02b823bcc4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/2c1f87379b34414a9bc81bc915718b5b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/44f97433c4fa468c912348b94e2a3012] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=36.0 K 2024-11-18T20:09:06,967 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:06,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/3ba09d0e9d7441098196c41445a8ce16, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb] 2024-11-18T20:09:06,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e7317ac4daa4c559d82df02b823bcc4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960542267 2024-11-18T20:09:06,967 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba09d0e9d7441098196c41445a8ce16, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960542267 2024-11-18T20:09:06,967 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c1f87379b34414a9bc81bc915718b5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731960542904 2024-11-18T20:09:06,968 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd49d9970b0f48489082496f0b3a0d30, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731960542904 2024-11-18T20:09:06,968 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f97433c4fa468c912348b94e2a3012, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960544531 2024-11-18T20:09:06,968 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 007f3ce258e0403cb4be82edb35b58fb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960544521 2024-11-18T20:09:06,974 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:06,975 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:06,975 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/18d08ba40f424bef926544b88e5ed873 is 50, key is test_row_0/B:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:06,978 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118e3d9cf821923425c98c361344db66837_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:06,980 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118e3d9cf821923425c98c361344db66837_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:06,981 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e3d9cf821923425c98c361344db66837_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:06,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742020_1196 (size=12663) 2024-11-18T20:09:06,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742021_1197 (size=4469) 2024-11-18T20:09:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:07,073 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:07,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:09:07,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111800bbb3e4e0f44a60a34732c8d8c27171_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960545705/Put/seqid=0 2024-11-18T20:09:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960607088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960607092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,094 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960607094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960607093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742022_1198 (size=17284) 2024-11-18T20:09:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960607202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960607202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960607202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:09:07,232 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:07,233 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-18T20:09:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:09:07,235 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:07,235 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:07,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:09:07,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,397 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/18d08ba40f424bef926544b88e5ed873 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/18d08ba40f424bef926544b88e5ed873 2024-11-18T20:09:07,397 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:07,397 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/719ce3334ab840e391066b56d670f412 is 175, key is test_row_0/A:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:07,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742023_1199 (size=31617) 2024-11-18T20:09:07,402 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into 18d08ba40f424bef926544b88e5ed873(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:07,402 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:07,402 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=13, startTime=1731960546965; duration=0sec 2024-11-18T20:09:07,403 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:07,403 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:09:07,403 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:07,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:07,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:09:07,404 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,404 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/9fecc4cd8dc04867bbe4caa1243ce673, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a0561827d33c426c970588678a86bd1a] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=36.0 K 2024-11-18T20:09:07,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2f8e3627e743bd8c9b5ac7ed1d7cab, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960542267 2024-11-18T20:09:07,405 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fecc4cd8dc04867bbe4caa1243ce673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731960542904 2024-11-18T20:09:07,405 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a0561827d33c426c970588678a86bd1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960544531 2024-11-18T20:09:07,411 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:07,412 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/a9d8e57dee7743a09f1bb5d6ba397b67 is 50, key is test_row_0/C:col10/1731960544531/Put/seqid=0 2024-11-18T20:09:07,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742024_1200 (size=12663) 2024-11-18T20:09:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960607412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960607412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960607412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:07,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111800bbb3e4e0f44a60a34732c8d8c27171_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111800bbb3e4e0f44a60a34732c8d8c27171_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:07,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/33388771dd6f4b77a7013c095f2b7bad, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:07,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/33388771dd6f4b77a7013c095f2b7bad is 175, key is test_row_0/A:col10/1731960545705/Put/seqid=0 2024-11-18T20:09:07,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742025_1201 (size=48389) 2024-11-18T20:09:07,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:09:07,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960607722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960607722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960607722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,806 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/719ce3334ab840e391066b56d670f412 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/719ce3334ab840e391066b56d670f412 2024-11-18T20:09:07,811 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into 719ce3334ab840e391066b56d670f412(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:07,811 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:07,811 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=13, startTime=1731960546965; duration=0sec 2024-11-18T20:09:07,811 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:07,811 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:09:07,820 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/a9d8e57dee7743a09f1bb5d6ba397b67 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a9d8e57dee7743a09f1bb5d6ba397b67 2024-11-18T20:09:07,825 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into a9d8e57dee7743a09f1bb5d6ba397b67(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:07,825 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:07,825 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=13, startTime=1731960546965; duration=0sec 2024-11-18T20:09:07,825 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:07,825 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:09:07,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:07,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:09:07,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960607901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:07,904 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:07,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/33388771dd6f4b77a7013c095f2b7bad 2024-11-18T20:09:07,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/bddde0f58c8544eeb4aa9b3a7ef92529 is 50, key is test_row_0/B:col10/1731960545705/Put/seqid=0 2024-11-18T20:09:07,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742026_1202 (size=12151) 2024-11-18T20:09:07,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:07,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:08,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:08,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960608232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960608232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960608232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:08,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:08,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:08,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:08,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/bddde0f58c8544eeb4aa9b3a7ef92529 2024-11-18T20:09:08,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/f6593b49307f45e68d270ef637bbee24 is 50, key is test_row_0/C:col10/1731960545705/Put/seqid=0 2024-11-18T20:09:08,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742027_1203 (size=12151) 2024-11-18T20:09:08,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/f6593b49307f45e68d270ef637bbee24 2024-11-18T20:09:08,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/33388771dd6f4b77a7013c095f2b7bad as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad 2024-11-18T20:09:08,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad, entries=250, sequenceid=237, filesize=47.3 K 2024-11-18T20:09:08,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/bddde0f58c8544eeb4aa9b3a7ef92529 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bddde0f58c8544eeb4aa9b3a7ef92529 2024-11-18T20:09:08,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bddde0f58c8544eeb4aa9b3a7ef92529, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T20:09:08,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/f6593b49307f45e68d270ef637bbee24 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/f6593b49307f45e68d270ef637bbee24 2024-11-18T20:09:08,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/f6593b49307f45e68d270ef637bbee24, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T20:09:08,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 43f9b782225aaf40f26cb68658ea7b20 in 1284ms, sequenceid=237, compaction requested=false 2024-11-18T20:09:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:09:08,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:09:08,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:08,462 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:09:08,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ec0084284e5949c892307571b793d34b_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960547092/Put/seqid=0 2024-11-18T20:09:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742028_1204 (size=12304) 2024-11-18T20:09:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:08,477 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ec0084284e5949c892307571b793d34b_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ec0084284e5949c892307571b793d34b_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:08,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/340e919ff64f4dc48068ac57a9db0c31, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/340e919ff64f4dc48068ac57a9db0c31 is 175, key is test_row_0/A:col10/1731960547092/Put/seqid=0 2024-11-18T20:09:08,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742029_1205 (size=31105) 2024-11-18T20:09:08,887 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/340e919ff64f4dc48068ac57a9db0c31 2024-11-18T20:09:08,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/397da392d00f4f998543db95cc8a2534 is 50, key is test_row_0/B:col10/1731960547092/Put/seqid=0 2024-11-18T20:09:08,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742030_1206 (size=12151) 2024-11-18T20:09:08,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/397da392d00f4f998543db95cc8a2534 2024-11-18T20:09:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/2d4ab6d7b47442109d21b96544448a0e is 50, key is test_row_0/C:col10/1731960547092/Put/seqid=0 2024-11-18T20:09:08,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742031_1207 (size=12151) 2024-11-18T20:09:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:09,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:09,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960609314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960609314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960609315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,343 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/2d4ab6d7b47442109d21b96544448a0e 2024-11-18T20:09:09,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/340e919ff64f4dc48068ac57a9db0c31 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31 2024-11-18T20:09:09,353 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31, entries=150, sequenceid=254, filesize=30.4 K 2024-11-18T20:09:09,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/397da392d00f4f998543db95cc8a2534 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/397da392d00f4f998543db95cc8a2534 2024-11-18T20:09:09,360 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/397da392d00f4f998543db95cc8a2534, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T20:09:09,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/2d4ab6d7b47442109d21b96544448a0e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/2d4ab6d7b47442109d21b96544448a0e 2024-11-18T20:09:09,368 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/2d4ab6d7b47442109d21b96544448a0e, entries=150, sequenceid=254, filesize=11.9 K 2024-11-18T20:09:09,369 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 43f9b782225aaf40f26cb68658ea7b20 in 907ms, sequenceid=254, compaction requested=true 2024-11-18T20:09:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-18T20:09:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-18T20:09:09,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-18T20:09:09,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1350 sec 2024-11-18T20:09:09,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1400 sec 2024-11-18T20:09:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:09:09,382 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:09,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-18T20:09:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:09,385 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:09,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:09,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:09,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:09,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:09:09,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186ad8a2d7c06a48ed9a4e5c87e7d7cc8f_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:09,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742032_1208 (size=14994) 2024-11-18T20:09:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960609431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960609440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960609441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186ad8a2d7c06a48ed9a4e5c87e7d7cc8f_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186ad8a2d7c06a48ed9a4e5c87e7d7cc8f_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,449 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/1e2d1d0a1d2f405889e8da8040efc233, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:09,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/1e2d1d0a1d2f405889e8da8040efc233 is 175, key is test_row_0/A:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:09,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742033_1209 (size=39949) 2024-11-18T20:09:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:09,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960609552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960609552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960609552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:09,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960609762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960609762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960609762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:09,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:09,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/1e2d1d0a1d2f405889e8da8040efc233 2024-11-18T20:09:09,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/ff681ed2a9c04f24bc8e388e51b65dea is 50, key is test_row_0/B:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:09,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960609923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:09,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742034_1210 (size=12301) 2024-11-18T20:09:09,931 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:09,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/ff681ed2a9c04f24bc8e388e51b65dea 2024-11-18T20:09:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:09,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/28a5e8d8c0f947d3b17f22386b34bddd is 50, key is test_row_0/C:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:09,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742035_1211 (size=12301) 2024-11-18T20:09:10,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960610072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960610074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960610074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:10,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/28a5e8d8c0f947d3b17f22386b34bddd 2024-11-18T20:09:10,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/1e2d1d0a1d2f405889e8da8040efc233 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233 2024-11-18T20:09:10,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233, entries=200, sequenceid=279, filesize=39.0 K 2024-11-18T20:09:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/ff681ed2a9c04f24bc8e388e51b65dea as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/ff681ed2a9c04f24bc8e388e51b65dea 2024-11-18T20:09:10,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/ff681ed2a9c04f24bc8e388e51b65dea, entries=150, sequenceid=279, filesize=12.0 K 2024-11-18T20:09:10,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/28a5e8d8c0f947d3b17f22386b34bddd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/28a5e8d8c0f947d3b17f22386b34bddd 2024-11-18T20:09:10,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/28a5e8d8c0f947d3b17f22386b34bddd, entries=150, sequenceid=279, filesize=12.0 K 2024-11-18T20:09:10,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 43f9b782225aaf40f26cb68658ea7b20 in 999ms, sequenceid=279, compaction requested=true 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:10,423 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:10,423 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:10,425 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:10,425 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:09:10,425 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,425 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/18d08ba40f424bef926544b88e5ed873, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bddde0f58c8544eeb4aa9b3a7ef92529, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/397da392d00f4f998543db95cc8a2534, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/ff681ed2a9c04f24bc8e388e51b65dea] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=48.1 K 2024-11-18T20:09:10,428 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151060 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:10,428 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:09:10,428 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,429 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/719ce3334ab840e391066b56d670f412, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=147.5 K 2024-11-18T20:09:10,429 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,429 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/719ce3334ab840e391066b56d670f412, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233] 2024-11-18T20:09:10,429 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d08ba40f424bef926544b88e5ed873, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960544531 2024-11-18T20:09:10,430 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719ce3334ab840e391066b56d670f412, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960544531 2024-11-18T20:09:10,430 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting bddde0f58c8544eeb4aa9b3a7ef92529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960545705 2024-11-18T20:09:10,430 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33388771dd6f4b77a7013c095f2b7bad, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960545705 2024-11-18T20:09:10,431 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 397da392d00f4f998543db95cc8a2534, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731960547081 2024-11-18T20:09:10,431 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 340e919ff64f4dc48068ac57a9db0c31, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731960547081 2024-11-18T20:09:10,431 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ff681ed2a9c04f24bc8e388e51b65dea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731960549303 2024-11-18T20:09:10,431 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2d1d0a1d2f405889e8da8040efc233, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731960549255 2024-11-18T20:09:10,444 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:10,445 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:10,446 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/76f8fc1bd036459f8e3168a69ac6805a is 50, key is test_row_0/B:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:10,450 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ea77c3d9e3d540ef8ca7fff85a6f70ae_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:10,453 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ea77c3d9e3d540ef8ca7fff85a6f70ae_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:10,453 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ea77c3d9e3d540ef8ca7fff85a6f70ae_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:10,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:09:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,482 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:09:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742036_1212 (size=4469) 2024-11-18T20:09:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:10,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742037_1213 (size=12949) 2024-11-18T20:09:10,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c3fbd0f08d54401c99dc924e7a33cd4e_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960549430/Put/seqid=0 2024-11-18T20:09:10,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742038_1214 (size=12454) 2024-11-18T20:09:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:10,563 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c3fbd0f08d54401c99dc924e7a33cd4e_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c3fbd0f08d54401c99dc924e7a33cd4e_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/0d4a90b695834e70bfb794ddfe300a9f, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/0d4a90b695834e70bfb794ddfe300a9f is 175, key is test_row_0/A:col10/1731960549430/Put/seqid=0 2024-11-18T20:09:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:10,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:10,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742039_1215 (size=31255) 2024-11-18T20:09:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960610670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960610672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960610680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960610782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960610782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960610792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,919 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#178 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:10,919 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/0c62b5c34d3942f99043638134437cb4 is 175, key is test_row_0/A:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:10,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742040_1216 (size=31903) 2024-11-18T20:09:10,927 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/0c62b5c34d3942f99043638134437cb4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0c62b5c34d3942f99043638134437cb4 2024-11-18T20:09:10,932 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into 0c62b5c34d3942f99043638134437cb4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:10,932 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:10,932 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=12, startTime=1731960550423; duration=0sec 2024-11-18T20:09:10,932 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:10,932 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:09:10,932 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:10,934 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:10,934 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:09:10,934 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:10,935 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a9d8e57dee7743a09f1bb5d6ba397b67, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/f6593b49307f45e68d270ef637bbee24, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/2d4ab6d7b47442109d21b96544448a0e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/28a5e8d8c0f947d3b17f22386b34bddd] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=48.1 K 2024-11-18T20:09:10,935 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/76f8fc1bd036459f8e3168a69ac6805a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/76f8fc1bd036459f8e3168a69ac6805a 2024-11-18T20:09:10,935 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d8e57dee7743a09f1bb5d6ba397b67, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960544531 2024-11-18T20:09:10,936 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6593b49307f45e68d270ef637bbee24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960545705 2024-11-18T20:09:10,936 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d4ab6d7b47442109d21b96544448a0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731960547081 2024-11-18T20:09:10,937 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a5e8d8c0f947d3b17f22386b34bddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731960549303 2024-11-18T20:09:10,941 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into 76f8fc1bd036459f8e3168a69ac6805a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:10,941 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:10,941 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=12, startTime=1731960550423; duration=0sec 2024-11-18T20:09:10,941 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:10,941 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:09:10,950 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:10,950 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/3de665cbb5fc485a978d09680b1ec9f2 is 50, key is test_row_0/C:col10/1731960549423/Put/seqid=0 2024-11-18T20:09:10,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742041_1217 (size=12949) 2024-11-18T20:09:10,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960610991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960610992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:10,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:10,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,001 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/0d4a90b695834e70bfb794ddfe300a9f 2024-11-18T20:09:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960611015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a5ffe112998d48179dc5d4573850ebff is 50, key is test_row_0/B:col10/1731960549430/Put/seqid=0 2024-11-18T20:09:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742042_1218 (size=12301) 2024-11-18T20:09:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45326 deadline: 1731960611122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,123 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960611302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960611302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960611322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,361 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/3de665cbb5fc485a978d09680b1ec9f2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3de665cbb5fc485a978d09680b1ec9f2 2024-11-18T20:09:11,367 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into 3de665cbb5fc485a978d09680b1ec9f2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:11,367 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:11,367 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=12, startTime=1731960550423; duration=0sec 2024-11-18T20:09:11,367 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:11,367 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:09:11,427 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a5ffe112998d48179dc5d4573850ebff 2024-11-18T20:09:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/69b86c2d4e5f406a94deccd3444b3a56 is 50, key is test_row_0/C:col10/1731960549430/Put/seqid=0 2024-11-18T20:09:11,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742043_1219 (size=12301) 2024-11-18T20:09:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:11,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960611812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960611812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960611832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:11,851 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/69b86c2d4e5f406a94deccd3444b3a56 2024-11-18T20:09:11,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/0d4a90b695834e70bfb794ddfe300a9f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f 2024-11-18T20:09:11,861 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f, entries=150, sequenceid=290, filesize=30.5 K 2024-11-18T20:09:11,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/a5ffe112998d48179dc5d4573850ebff as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5ffe112998d48179dc5d4573850ebff 2024-11-18T20:09:11,866 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5ffe112998d48179dc5d4573850ebff, entries=150, sequenceid=290, filesize=12.0 K 2024-11-18T20:09:11,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/69b86c2d4e5f406a94deccd3444b3a56 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/69b86c2d4e5f406a94deccd3444b3a56 2024-11-18T20:09:11,872 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/69b86c2d4e5f406a94deccd3444b3a56, entries=150, sequenceid=290, filesize=12.0 K 2024-11-18T20:09:11,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 43f9b782225aaf40f26cb68658ea7b20 in 1391ms, sequenceid=290, compaction requested=false 2024-11-18T20:09:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-18T20:09:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-18T20:09:11,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-18T20:09:11,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-11-18T20:09:11,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4930 sec 2024-11-18T20:09:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:12,826 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:12,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:09:12,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d7072dd3bfa0427084152e6835f9d77c_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742044_1220 (size=14994) 2024-11-18T20:09:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960612840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960612842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45250 deadline: 1731960612842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:12,845 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:12,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960612952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960612952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960613162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960613162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:13,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d7072dd3bfa0427084152e6835f9d77c_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d7072dd3bfa0427084152e6835f9d77c_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:13,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/bcd77e2e69204f48b3e94b7bde3c37be, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:13,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/bcd77e2e69204f48b3e94b7bde3c37be is 175, key is test_row_0/A:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:13,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742045_1221 (size=39949) 2024-11-18T20:09:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960613472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960613472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:09:13,552 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:13,552 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-18T20:09:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:13,554 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:13,555 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:13,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:13,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/bcd77e2e69204f48b3e94b7bde3c37be 2024-11-18T20:09:13,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/83560a32d7f14f718dc16335246be860 is 50, key is test_row_0/B:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:13,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742046_1222 (size=12301) 2024-11-18T20:09:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:13,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:13,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:13,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:13,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:13,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:13,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:13,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:13,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45324 deadline: 1731960613972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,973 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 43f9b782225aaf40f26cb68658ea7b20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731960613982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45316 deadline: 1731960613982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:13,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:14,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/83560a32d7f14f718dc16335246be860 2024-11-18T20:09:14,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/55cf83369ade4b62adeedb9af07cfa01 is 50, key is test_row_0/C:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742047_1223 (size=12301) 2024-11-18T20:09:14,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:14,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/55cf83369ade4b62adeedb9af07cfa01 2024-11-18T20:09:14,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/bcd77e2e69204f48b3e94b7bde3c37be as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be 2024-11-18T20:09:14,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be, entries=200, sequenceid=319, filesize=39.0 K 2024-11-18T20:09:14,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/83560a32d7f14f718dc16335246be860 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/83560a32d7f14f718dc16335246be860 2024-11-18T20:09:14,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/83560a32d7f14f718dc16335246be860, entries=150, sequenceid=319, filesize=12.0 K 2024-11-18T20:09:14,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/55cf83369ade4b62adeedb9af07cfa01 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/55cf83369ade4b62adeedb9af07cfa01 2024-11-18T20:09:14,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/55cf83369ade4b62adeedb9af07cfa01, entries=150, sequenceid=319, filesize=12.0 K 2024-11-18T20:09:14,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 43f9b782225aaf40f26cb68658ea7b20 in 1666ms, sequenceid=319, compaction requested=true 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:14,493 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:14,493 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43f9b782225aaf40f26cb68658ea7b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:14,494 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:14,494 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/A is initiating minor compaction (all files) 2024-11-18T20:09:14,494 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/A in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,494 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0c62b5c34d3942f99043638134437cb4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=100.7 K 2024-11-18T20:09:14,494 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,494 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0c62b5c34d3942f99043638134437cb4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be] 2024-11-18T20:09:14,494 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:14,495 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/B is initiating minor compaction (all files) 2024-11-18T20:09:14,495 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/B in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,495 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/76f8fc1bd036459f8e3168a69ac6805a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5ffe112998d48179dc5d4573850ebff, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/83560a32d7f14f718dc16335246be860] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=36.7 K 2024-11-18T20:09:14,495 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c62b5c34d3942f99043638134437cb4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731960549303 2024-11-18T20:09:14,495 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f8fc1bd036459f8e3168a69ac6805a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731960549303 2024-11-18T20:09:14,495 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d4a90b695834e70bfb794ddfe300a9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731960549429 2024-11-18T20:09:14,495 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ffe112998d48179dc5d4573850ebff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731960549429 2024-11-18T20:09:14,496 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 83560a32d7f14f718dc16335246be860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731960550668 2024-11-18T20:09:14,496 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd77e2e69204f48b3e94b7bde3c37be, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731960550667 2024-11-18T20:09:14,502 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:14,503 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:14,503 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e5b3033a90644a8eb1e5df3550886ede is 50, key is test_row_0/B:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:14,513 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111877ec607b5bba4819b37b85a463608a67_43f9b782225aaf40f26cb68658ea7b20 store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:14,515 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111877ec607b5bba4819b37b85a463608a67_43f9b782225aaf40f26cb68658ea7b20, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:14,515 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111877ec607b5bba4819b37b85a463608a67_43f9b782225aaf40f26cb68658ea7b20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:14,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742048_1224 (size=13051) 2024-11-18T20:09:14,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742049_1225 (size=4469) 2024-11-18T20:09:14,531 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#A#compaction#187 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:14,532 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/41de966c48a840b5aa3956f26ae88db1 is 175, key is test_row_0/A:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:14,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742050_1226 (size=32005) 2024-11-18T20:09:14,541 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/41de966c48a840b5aa3956f26ae88db1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/41de966c48a840b5aa3956f26ae88db1 2024-11-18T20:09:14,546 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/A of 43f9b782225aaf40f26cb68658ea7b20 into 41de966c48a840b5aa3956f26ae88db1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:14,546 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:14,546 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/A, priority=13, startTime=1731960554493; duration=0sec 2024-11-18T20:09:14,546 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:14,546 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:A 2024-11-18T20:09:14,546 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:14,547 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:14,547 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 43f9b782225aaf40f26cb68658ea7b20/C is initiating minor compaction (all files) 2024-11-18T20:09:14,547 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43f9b782225aaf40f26cb68658ea7b20/C in TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,547 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3de665cbb5fc485a978d09680b1ec9f2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/69b86c2d4e5f406a94deccd3444b3a56, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/55cf83369ade4b62adeedb9af07cfa01] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp, totalSize=36.7 K 2024-11-18T20:09:14,548 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de665cbb5fc485a978d09680b1ec9f2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731960549303 2024-11-18T20:09:14,548 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b86c2d4e5f406a94deccd3444b3a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731960549429 2024-11-18T20:09:14,548 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55cf83369ade4b62adeedb9af07cfa01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731960550668 2024-11-18T20:09:14,556 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43f9b782225aaf40f26cb68658ea7b20#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:14,557 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/7ea299b21b7747a5ad7b357b7ef7190e is 50, key is test_row_0/C:col10/1731960550668/Put/seqid=0 2024-11-18T20:09:14,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742051_1227 (size=13051) 2024-11-18T20:09:14,572 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/7ea299b21b7747a5ad7b357b7ef7190e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7ea299b21b7747a5ad7b357b7ef7190e 2024-11-18T20:09:14,579 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/C of 43f9b782225aaf40f26cb68658ea7b20 into 7ea299b21b7747a5ad7b357b7ef7190e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:14,579 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:14,579 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/C, priority=13, startTime=1731960554493; duration=0sec 2024-11-18T20:09:14,579 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:14,579 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:C 2024-11-18T20:09:14,580 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-18T20:09:14,580 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-18T20:09:14,580 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:14,580 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:14,580 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,580 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,580 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:14,581 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:14,584 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-18T20:09:14,584 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:14,584 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,584 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:14,586 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-18T20:09:14,586 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:14,586 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,586 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:14,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:09:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:14,627 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-18T20:09:14,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111856db158560524aad81588b3fc911aefd_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960552841/Put/seqid=0 2024-11-18T20:09:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742052_1228 (size=12454) 2024-11-18T20:09:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:14,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. as already flushing 2024-11-18T20:09:14,867 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-18T20:09:14,867 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:14,867 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,867 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:14,933 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e5b3033a90644a8eb1e5df3550886ede as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e5b3033a90644a8eb1e5df3550886ede 2024-11-18T20:09:14,940 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43f9b782225aaf40f26cb68658ea7b20/B of 43f9b782225aaf40f26cb68658ea7b20 into e5b3033a90644a8eb1e5df3550886ede(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:14,940 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:14,940 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., storeName=43f9b782225aaf40f26cb68658ea7b20/B, priority=13, startTime=1731960554493; duration=0sec 2024-11-18T20:09:14,940 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:14,940 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43f9b782225aaf40f26cb68658ea7b20:B 2024-11-18T20:09:14,993 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-18T20:09:14,993 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:14,993 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,993 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:14,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:15,003 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-18T20:09:15,003 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:15,003 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:15,003 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:15,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:15,045 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111856db158560524aad81588b3fc911aefd_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111856db158560524aad81588b3fc911aefd_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/13fb3a605e6743d8a37c605ac6aa218f, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/13fb3a605e6743d8a37c605ac6aa218f is 175, key is test_row_0/A:col10/1731960552841/Put/seqid=0 2024-11-18T20:09:15,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742053_1229 (size=31255) 2024-11-18T20:09:15,453 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/13fb3a605e6743d8a37c605ac6aa218f 2024-11-18T20:09:15,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/4336bd9c6b064c05b6aed41b73f081af is 50, key is test_row_0/B:col10/1731960552841/Put/seqid=0 2024-11-18T20:09:15,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742054_1230 (size=12301) 2024-11-18T20:09:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:15,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:09:15,901 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/4336bd9c6b064c05b6aed41b73f081af 2024-11-18T20:09:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/bd2e5811e2594a938b05e2c5052eaba2 is 50, key is test_row_0/C:col10/1731960552841/Put/seqid=0 2024-11-18T20:09:15,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742055_1231 (size=12301) 2024-11-18T20:09:16,313 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/bd2e5811e2594a938b05e2c5052eaba2 2024-11-18T20:09:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/13fb3a605e6743d8a37c605ac6aa218f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/13fb3a605e6743d8a37c605ac6aa218f 2024-11-18T20:09:16,325 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/13fb3a605e6743d8a37c605ac6aa218f, entries=150, sequenceid=331, filesize=30.5 K 2024-11-18T20:09:16,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/4336bd9c6b064c05b6aed41b73f081af as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/4336bd9c6b064c05b6aed41b73f081af 2024-11-18T20:09:16,330 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/4336bd9c6b064c05b6aed41b73f081af, entries=150, sequenceid=331, filesize=12.0 K 2024-11-18T20:09:16,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/bd2e5811e2594a938b05e2c5052eaba2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/bd2e5811e2594a938b05e2c5052eaba2 2024-11-18T20:09:16,335 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/bd2e5811e2594a938b05e2c5052eaba2, entries=150, sequenceid=331, filesize=12.0 K 2024-11-18T20:09:16,336 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=20.13 KB/20610 for 43f9b782225aaf40f26cb68658ea7b20 in 1708ms, sequenceid=331, compaction requested=false 2024-11-18T20:09:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 43f9b782225aaf40f26cb68658ea7b20: 2024-11-18T20:09:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-18T20:09:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-18T20:09:16,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T20:09:16,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7820 sec 2024-11-18T20:09:16,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.7860 sec 2024-11-18T20:09:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:09:17,714 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:21,216 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-18T20:09:21,216 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:21,216 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:21,216 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:21,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:24,047 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-18T20:09:24,047 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:24,047 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:24,047 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:24,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4026 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4031 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1704 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5105 rows 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1709 2024-11-18T20:09:24,048 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5122 rows 2024-11-18T20:09:24,049 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:09:24,049 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:09:24,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:24,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:24,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:24,050 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:09:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:09:24,054 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960564053"}]},"ts":"1731960564053"} 2024-11-18T20:09:24,055 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:09:24,055 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:09:24,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:09:24,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, UNASSIGN}] 2024-11-18T20:09:24,058 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, UNASSIGN 2024-11-18T20:09:24,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:24,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, UNASSIGN because future has completed 2024-11-18T20:09:24,061 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:09:24,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:09:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:09:24,215 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:24,215 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:09:24,215 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 43f9b782225aaf40f26cb68658ea7b20, disabling compactions & flushes 2024-11-18T20:09:24,215 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:24,215 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:24,216 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. after waiting 0 ms 2024-11-18T20:09:24,216 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:24,216 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 43f9b782225aaf40f26cb68658ea7b20 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:09:24,229 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189bfae1c61d424ea0a9a68d4e6faa75c4_43f9b782225aaf40f26cb68658ea7b20 is 50, key is test_row_0/A:col10/1731960564043/Put/seqid=0 2024-11-18T20:09:24,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742056_1232 (size=9914) 2024-11-18T20:09:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:09:24,637 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:24,647 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189bfae1c61d424ea0a9a68d4e6faa75c4_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189bfae1c61d424ea0a9a68d4e6faa75c4_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:24,649 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/16e53a61cda34e03b1d31bc26c6064c0, store: [table=TestAcidGuarantees family=A region=43f9b782225aaf40f26cb68658ea7b20] 2024-11-18T20:09:24,650 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/16e53a61cda34e03b1d31bc26c6064c0 is 175, key is test_row_0/A:col10/1731960564043/Put/seqid=0 2024-11-18T20:09:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742057_1233 (size=22561) 2024-11-18T20:09:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:09:25,057 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/16e53a61cda34e03b1d31bc26c6064c0 2024-11-18T20:09:25,072 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e9815b70856b4e7db4ee6eefed337e05 is 50, key is test_row_0/B:col10/1731960564043/Put/seqid=0 2024-11-18T20:09:25,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742058_1234 (size=9857) 2024-11-18T20:09:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:09:25,477 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e9815b70856b4e7db4ee6eefed337e05 2024-11-18T20:09:25,484 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/ef7070975b0642cd92df62588519c97c is 50, key is test_row_0/C:col10/1731960564043/Put/seqid=0 2024-11-18T20:09:25,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742059_1235 (size=9857) 2024-11-18T20:09:25,889 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/ef7070975b0642cd92df62588519c97c 2024-11-18T20:09:25,931 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/A/16e53a61cda34e03b1d31bc26c6064c0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/16e53a61cda34e03b1d31bc26c6064c0 2024-11-18T20:09:25,937 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/16e53a61cda34e03b1d31bc26c6064c0, entries=100, sequenceid=340, filesize=22.0 K 2024-11-18T20:09:25,938 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/B/e9815b70856b4e7db4ee6eefed337e05 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e9815b70856b4e7db4ee6eefed337e05 2024-11-18T20:09:25,942 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e9815b70856b4e7db4ee6eefed337e05, entries=100, sequenceid=340, filesize=9.6 K 2024-11-18T20:09:25,943 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/.tmp/C/ef7070975b0642cd92df62588519c97c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ef7070975b0642cd92df62588519c97c 2024-11-18T20:09:25,947 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ef7070975b0642cd92df62588519c97c, entries=100, sequenceid=340, filesize=9.6 K 2024-11-18T20:09:25,948 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 43f9b782225aaf40f26cb68658ea7b20 in 1732ms, sequenceid=340, compaction requested=true 2024-11-18T20:09:25,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5f662595dbc04fb6a027ddd3516f7319, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d6e25af9fc84472ab7161e723244fb24, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fb76783522dd4cf5be70302d94f193c0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/3ba09d0e9d7441098196c41445a8ce16, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/719ce3334ab840e391066b56d670f412, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0c62b5c34d3942f99043638134437cb4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be] to archive 2024-11-18T20:09:25,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:09:25,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d64be8a7a7ef47ffb04cbe652440c5ad 2024-11-18T20:09:25,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a0bb593437174566be4151e118a48f15 2024-11-18T20:09:25,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5f662595dbc04fb6a027ddd3516f7319 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5f662595dbc04fb6a027ddd3516f7319 2024-11-18T20:09:25,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/5483cbfef96b49b0a00ab565a84871cb 2024-11-18T20:09:25,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/e122017e7aa940d6a12ee6d277457ff5 2024-11-18T20:09:25,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/66a3a1576d074d7da2f1b0ffe94f2d30 2024-11-18T20:09:25,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d6e25af9fc84472ab7161e723244fb24 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/d6e25af9fc84472ab7161e723244fb24 2024-11-18T20:09:25,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/eb699d1b43854f53a85cc754d4780abb 2024-11-18T20:09:25,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/2e06aec5a72948dbb97abcb47a7f2e4c 2024-11-18T20:09:25,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fb76783522dd4cf5be70302d94f193c0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fb76783522dd4cf5be70302d94f193c0 2024-11-18T20:09:25,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/a6c927d7c23f4985a6e7f13314356a7d 2024-11-18T20:09:25,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/ca8dd678bf7c432c93a51f187099f486 2024-11-18T20:09:25,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/3ba09d0e9d7441098196c41445a8ce16 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/3ba09d0e9d7441098196c41445a8ce16 2024-11-18T20:09:25,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/fd49d9970b0f48489082496f0b3a0d30 2024-11-18T20:09:25,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/007f3ce258e0403cb4be82edb35b58fb 2024-11-18T20:09:25,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/719ce3334ab840e391066b56d670f412 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/719ce3334ab840e391066b56d670f412 2024-11-18T20:09:25,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/33388771dd6f4b77a7013c095f2b7bad 2024-11-18T20:09:25,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/340e919ff64f4dc48068ac57a9db0c31 2024-11-18T20:09:25,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/1e2d1d0a1d2f405889e8da8040efc233 2024-11-18T20:09:25,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0c62b5c34d3942f99043638134437cb4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0c62b5c34d3942f99043638134437cb4 2024-11-18T20:09:25,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/0d4a90b695834e70bfb794ddfe300a9f 2024-11-18T20:09:25,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/bcd77e2e69204f48b3e94b7bde3c37be 2024-11-18T20:09:25,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5121bcf11c0446fa09c7bbf93d75447, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/090b4edb6c6c44dcb7062d16537d2b31, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a4a68020051e44a9bfecf34e3f314985, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/099d9817c680459d9dd994be3b833bd7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/b8f697be5f714b9da4bba6e86c3880f9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e679ad48ce314b4f97b8f7ad3bb1b63d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a03d5f2cbb8046168d486b99b99e4133, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5934750f6e814ae6aac5d4fdd130c336, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/6e2a4dd896da447c83de339f96dc80a5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/3c58a281d69d4e56ad8c0fc9b77bfccf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bf77496086294a2f82b752b417835c18, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5e7317ac4daa4c559d82df02b823bcc4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/405a68480ea14cdf99b06ee09db54a93, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/2c1f87379b34414a9bc81bc915718b5b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/18d08ba40f424bef926544b88e5ed873, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/44f97433c4fa468c912348b94e2a3012, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bddde0f58c8544eeb4aa9b3a7ef92529, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/397da392d00f4f998543db95cc8a2534, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/76f8fc1bd036459f8e3168a69ac6805a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/ff681ed2a9c04f24bc8e388e51b65dea, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5ffe112998d48179dc5d4573850ebff, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/83560a32d7f14f718dc16335246be860] to archive 2024-11-18T20:09:25,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:09:25,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5121bcf11c0446fa09c7bbf93d75447 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5121bcf11c0446fa09c7bbf93d75447 2024-11-18T20:09:25,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/090b4edb6c6c44dcb7062d16537d2b31 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/090b4edb6c6c44dcb7062d16537d2b31 2024-11-18T20:09:25,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a4a68020051e44a9bfecf34e3f314985 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a4a68020051e44a9bfecf34e3f314985 2024-11-18T20:09:25,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/099d9817c680459d9dd994be3b833bd7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/099d9817c680459d9dd994be3b833bd7 2024-11-18T20:09:25,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/b8f697be5f714b9da4bba6e86c3880f9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/b8f697be5f714b9da4bba6e86c3880f9 2024-11-18T20:09:25,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e679ad48ce314b4f97b8f7ad3bb1b63d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e679ad48ce314b4f97b8f7ad3bb1b63d 2024-11-18T20:09:25,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a03d5f2cbb8046168d486b99b99e4133 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a03d5f2cbb8046168d486b99b99e4133 2024-11-18T20:09:25,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5934750f6e814ae6aac5d4fdd130c336 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5934750f6e814ae6aac5d4fdd130c336 2024-11-18T20:09:25,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/6e2a4dd896da447c83de339f96dc80a5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/6e2a4dd896da447c83de339f96dc80a5 2024-11-18T20:09:25,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/3c58a281d69d4e56ad8c0fc9b77bfccf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/3c58a281d69d4e56ad8c0fc9b77bfccf 2024-11-18T20:09:25,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bf77496086294a2f82b752b417835c18 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bf77496086294a2f82b752b417835c18 2024-11-18T20:09:25,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5e7317ac4daa4c559d82df02b823bcc4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/5e7317ac4daa4c559d82df02b823bcc4 2024-11-18T20:09:25,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/405a68480ea14cdf99b06ee09db54a93 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/405a68480ea14cdf99b06ee09db54a93 2024-11-18T20:09:25,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/2c1f87379b34414a9bc81bc915718b5b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/2c1f87379b34414a9bc81bc915718b5b 2024-11-18T20:09:25,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/18d08ba40f424bef926544b88e5ed873 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/18d08ba40f424bef926544b88e5ed873 2024-11-18T20:09:25,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/44f97433c4fa468c912348b94e2a3012 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/44f97433c4fa468c912348b94e2a3012 2024-11-18T20:09:25,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bddde0f58c8544eeb4aa9b3a7ef92529 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/bddde0f58c8544eeb4aa9b3a7ef92529 2024-11-18T20:09:25,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/397da392d00f4f998543db95cc8a2534 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/397da392d00f4f998543db95cc8a2534 2024-11-18T20:09:25,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/76f8fc1bd036459f8e3168a69ac6805a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/76f8fc1bd036459f8e3168a69ac6805a 2024-11-18T20:09:25,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/ff681ed2a9c04f24bc8e388e51b65dea to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/ff681ed2a9c04f24bc8e388e51b65dea 2024-11-18T20:09:25,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5ffe112998d48179dc5d4573850ebff to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/a5ffe112998d48179dc5d4573850ebff 2024-11-18T20:09:25,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/83560a32d7f14f718dc16335246be860 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/83560a32d7f14f718dc16335246be860 2024-11-18T20:09:25,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/c63cc2f2588a45e5b0e379b99aa038f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/29cf8325cfd94330aff38c440cffb644, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/5ea0bb46a3374fdcad8b3ca229d89db0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/578e77c5505a4ed3b5c04653e0034b96, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/0097d7689f0d48dc8f6ad23f2e94b83a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/e7d9dba8ddc64d99bc135093ee229160, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3c42d44722ce49c7811d2da81cd6e428, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4e4d92a0f502445d98fb551b5ea2b9d9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/990a914f9e054861be6003bba2a6e75f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/d3e06a8e1f0244628dcc026c5ae1db7e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ab9e73912c0c44d7bdae5f96506d9366, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7d1258d9b0b04e58b2c618f78144901f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/9fecc4cd8dc04867bbe4caa1243ce673, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a9d8e57dee7743a09f1bb5d6ba397b67, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a0561827d33c426c970588678a86bd1a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/f6593b49307f45e68d270ef637bbee24, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/2d4ab6d7b47442109d21b96544448a0e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3de665cbb5fc485a978d09680b1ec9f2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/28a5e8d8c0f947d3b17f22386b34bddd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/69b86c2d4e5f406a94deccd3444b3a56, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/55cf83369ade4b62adeedb9af07cfa01] to archive 2024-11-18T20:09:25,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:09:25,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/c63cc2f2588a45e5b0e379b99aa038f8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/c63cc2f2588a45e5b0e379b99aa038f8 2024-11-18T20:09:25,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/29cf8325cfd94330aff38c440cffb644 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/29cf8325cfd94330aff38c440cffb644 2024-11-18T20:09:26,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/5ea0bb46a3374fdcad8b3ca229d89db0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/5ea0bb46a3374fdcad8b3ca229d89db0 2024-11-18T20:09:26,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/578e77c5505a4ed3b5c04653e0034b96 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/578e77c5505a4ed3b5c04653e0034b96 2024-11-18T20:09:26,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/0097d7689f0d48dc8f6ad23f2e94b83a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/0097d7689f0d48dc8f6ad23f2e94b83a 2024-11-18T20:09:26,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/e7d9dba8ddc64d99bc135093ee229160 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/e7d9dba8ddc64d99bc135093ee229160 2024-11-18T20:09:26,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3c42d44722ce49c7811d2da81cd6e428 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3c42d44722ce49c7811d2da81cd6e428 2024-11-18T20:09:26,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4e4d92a0f502445d98fb551b5ea2b9d9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4e4d92a0f502445d98fb551b5ea2b9d9 2024-11-18T20:09:26,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/990a914f9e054861be6003bba2a6e75f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/990a914f9e054861be6003bba2a6e75f 2024-11-18T20:09:26,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/d3e06a8e1f0244628dcc026c5ae1db7e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/d3e06a8e1f0244628dcc026c5ae1db7e 2024-11-18T20:09:26,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ab9e73912c0c44d7bdae5f96506d9366 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ab9e73912c0c44d7bdae5f96506d9366 2024-11-18T20:09:26,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/4c2f8e3627e743bd8c9b5ac7ed1d7cab 2024-11-18T20:09:26,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7d1258d9b0b04e58b2c618f78144901f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7d1258d9b0b04e58b2c618f78144901f 2024-11-18T20:09:26,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/9fecc4cd8dc04867bbe4caa1243ce673 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/9fecc4cd8dc04867bbe4caa1243ce673 2024-11-18T20:09:26,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a9d8e57dee7743a09f1bb5d6ba397b67 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a9d8e57dee7743a09f1bb5d6ba397b67 2024-11-18T20:09:26,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a0561827d33c426c970588678a86bd1a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/a0561827d33c426c970588678a86bd1a 2024-11-18T20:09:26,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/f6593b49307f45e68d270ef637bbee24 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/f6593b49307f45e68d270ef637bbee24 2024-11-18T20:09:26,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/2d4ab6d7b47442109d21b96544448a0e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/2d4ab6d7b47442109d21b96544448a0e 2024-11-18T20:09:26,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3de665cbb5fc485a978d09680b1ec9f2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/3de665cbb5fc485a978d09680b1ec9f2 2024-11-18T20:09:26,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/28a5e8d8c0f947d3b17f22386b34bddd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/28a5e8d8c0f947d3b17f22386b34bddd 2024-11-18T20:09:26,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/69b86c2d4e5f406a94deccd3444b3a56 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/69b86c2d4e5f406a94deccd3444b3a56 2024-11-18T20:09:26,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/55cf83369ade4b62adeedb9af07cfa01 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/55cf83369ade4b62adeedb9af07cfa01 2024-11-18T20:09:26,020 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=4 2024-11-18T20:09:26,020 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. 2024-11-18T20:09:26,020 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 43f9b782225aaf40f26cb68658ea7b20: Waiting for close lock at 1731960564215Running coprocessor pre-close hooks at 1731960564215Disabling compacts and flushes for region at 1731960564215Disabling writes for close at 1731960564216 (+1 ms)Obtaining lock to block concurrent updates at 1731960564216Preparing flush snapshotting stores in 43f9b782225aaf40f26cb68658ea7b20 at 1731960564216Finished memstore snapshotting TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960564217 (+1 ms)Flushing stores of TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20. at 1731960564218 (+1 ms)Flushing 43f9b782225aaf40f26cb68658ea7b20/A: creating writer at 1731960564218Flushing mob file 43f9b782225aaf40f26cb68658ea7b20/A: appending metadata at 1731960564229 (+11 ms)Flushing mob file 43f9b782225aaf40f26cb68658ea7b20/A: closing flushed file at 1731960564229Flushing 43f9b782225aaf40f26cb68658ea7b20/A: appending metadata at 1731960564649 (+420 ms)Flushing 43f9b782225aaf40f26cb68658ea7b20/A: closing flushed file at 1731960564649Flushing 43f9b782225aaf40f26cb68658ea7b20/B: creating writer at 1731960565067 (+418 ms)Flushing 43f9b782225aaf40f26cb68658ea7b20/B: appending metadata at 1731960565072 (+5 ms)Flushing 43f9b782225aaf40f26cb68658ea7b20/B: closing flushed file at 1731960565072Flushing 43f9b782225aaf40f26cb68658ea7b20/C: creating writer at 1731960565481 (+409 ms)Flushing 43f9b782225aaf40f26cb68658ea7b20/C: appending metadata at 1731960565483 (+2 ms)Flushing 43f9b782225aaf40f26cb68658ea7b20/C: closing flushed file at 1731960565483Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734b2b52: reopening flushed file at 1731960565929 (+446 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56bbbd51: reopening flushed file at 1731960565937 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@236ccf30: reopening flushed file at 1731960565942 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 43f9b782225aaf40f26cb68658ea7b20 in 1732ms, sequenceid=340, compaction requested=true at 1731960565948 (+6 ms)Writing region close event to WAL at 1731960566017 (+69 ms)Running coprocessor post-close hooks at 1731960566020 (+3 ms)Closed at 1731960566020 2024-11-18T20:09:26,022 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,023 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=43f9b782225aaf40f26cb68658ea7b20, regionState=CLOSED 2024-11-18T20:09:26,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:09:26,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-18T20:09:26,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 43f9b782225aaf40f26cb68658ea7b20, server=d8f95bb18b58,42447,1731960497538 in 1.9630 sec 2024-11-18T20:09:26,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-18T20:09:26,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43f9b782225aaf40f26cb68658ea7b20, UNASSIGN in 1.9680 sec 2024-11-18T20:09:26,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T20:09:26,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9710 sec 2024-11-18T20:09:26,029 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960566029"}]},"ts":"1731960566029"} 2024-11-18T20:09:26,030 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:09:26,030 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:09:26,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9810 sec 2024-11-18T20:09:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:09:26,203 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:26,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:09:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,207 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:09:26,208 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,214 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,219 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/recovered.edits] 2024-11-18T20:09:26,222 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/13fb3a605e6743d8a37c605ac6aa218f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/13fb3a605e6743d8a37c605ac6aa218f 2024-11-18T20:09:26,224 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/16e53a61cda34e03b1d31bc26c6064c0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/16e53a61cda34e03b1d31bc26c6064c0 2024-11-18T20:09:26,225 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/41de966c48a840b5aa3956f26ae88db1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/A/41de966c48a840b5aa3956f26ae88db1 2024-11-18T20:09:26,227 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/4336bd9c6b064c05b6aed41b73f081af to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/4336bd9c6b064c05b6aed41b73f081af 2024-11-18T20:09:26,229 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e5b3033a90644a8eb1e5df3550886ede to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e5b3033a90644a8eb1e5df3550886ede 2024-11-18T20:09:26,230 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e9815b70856b4e7db4ee6eefed337e05 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/B/e9815b70856b4e7db4ee6eefed337e05 2024-11-18T20:09:26,232 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7ea299b21b7747a5ad7b357b7ef7190e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/7ea299b21b7747a5ad7b357b7ef7190e 2024-11-18T20:09:26,233 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/bd2e5811e2594a938b05e2c5052eaba2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/bd2e5811e2594a938b05e2c5052eaba2 2024-11-18T20:09:26,235 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ef7070975b0642cd92df62588519c97c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/C/ef7070975b0642cd92df62588519c97c 2024-11-18T20:09:26,237 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/recovered.edits/343.seqid to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20/recovered.edits/343.seqid 2024-11-18T20:09:26,238 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,238 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:09:26,238 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:09:26,239 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:09:26,243 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111800bbb3e4e0f44a60a34732c8d8c27171_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111800bbb3e4e0f44a60a34732c8d8c27171_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,244 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118087c678f266c4b68afb7e5d7842f7c26_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118087c678f266c4b68afb7e5d7842f7c26_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,245 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180e7212161d1b4101a5bbde7d7e867d00_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180e7212161d1b4101a5bbde7d7e867d00_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,246 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111839b772ea028e406689bfa4ef5c6a79e5_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111839b772ea028e406689bfa4ef5c6a79e5_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,248 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111856db158560524aad81588b3fc911aefd_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111856db158560524aad81588b3fc911aefd_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,249 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866fbf618b30f446c93c0d31a4f920334_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111866fbf618b30f446c93c0d31a4f920334_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,249 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186ad8a2d7c06a48ed9a4e5c87e7d7cc8f_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186ad8a2d7c06a48ed9a4e5c87e7d7cc8f_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,250 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118700c374ce00e462ead17d8bb5d19aa48_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118700c374ce00e462ead17d8bb5d19aa48_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,251 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111877ec4c5eeeff44d1b20d8be52a61cb22_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111877ec4c5eeeff44d1b20d8be52a61cb22_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,252 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189bfae1c61d424ea0a9a68d4e6faa75c4_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189bfae1c61d424ea0a9a68d4e6faa75c4_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,253 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a5f65a0b433b45b09c86ab8dcaa44eca_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a5f65a0b433b45b09c86ab8dcaa44eca_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,254 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b7d4f68d8f8142aba2ce83061ecf1a96_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b7d4f68d8f8142aba2ce83061ecf1a96_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,255 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bb51ae4bb822455c8eba423c02363c5c_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bb51ae4bb822455c8eba423c02363c5c_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,256 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bf332ba768fb4849b9d0b5d0c83efcfa_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bf332ba768fb4849b9d0b5d0c83efcfa_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,257 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c3fbd0f08d54401c99dc924e7a33cd4e_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c3fbd0f08d54401c99dc924e7a33cd4e_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,258 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d7072dd3bfa0427084152e6835f9d77c_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d7072dd3bfa0427084152e6835f9d77c_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,258 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ec0084284e5949c892307571b793d34b_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ec0084284e5949c892307571b793d34b_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,259 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee12f3ff853349f7b52a0d86df719029_43f9b782225aaf40f26cb68658ea7b20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ee12f3ff853349f7b52a0d86df719029_43f9b782225aaf40f26cb68658ea7b20 2024-11-18T20:09:26,260 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:09:26,261 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,264 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:09:26,266 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:09:26,267 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,267 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:09:26,267 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960566267"}]},"ts":"9223372036854775807"} 2024-11-18T20:09:26,269 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:09:26,269 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 43f9b782225aaf40f26cb68658ea7b20, NAME => 'TestAcidGuarantees,,1731960531390.43f9b782225aaf40f26cb68658ea7b20.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:09:26,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:09:26,270 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960566269"}]},"ts":"9223372036854775807"} 2024-11-18T20:09:26,271 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:09:26,272 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-18T20:09:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:09:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:09:26,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:26,321 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 238) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1190570139_22 at /127.0.0.1:58132 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 555), ProcessCount=11 (was 11), AvailableMemoryMB=1163 (was 1338) 2024-11-18T20:09:26,329 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=1163 2024-11-18T20:09:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:09:26,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:09:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:26,332 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:09:26,333 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:26,333 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-18T20:09:26,333 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:09:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:09:26,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742060_1236 (size=959) 2024-11-18T20:09:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:09:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:09:26,745 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:09:26,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742061_1237 (size=53) 2024-11-18T20:09:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:09:27,096 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:09:27,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:09:27,154 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9c697d531cc5d270f339498b2c3c441b, disabling compactions & flushes 2024-11-18T20:09:27,154 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,154 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,154 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. after waiting 0 ms 2024-11-18T20:09:27,154 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,154 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,154 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c697d531cc5d270f339498b2c3c441b: Waiting for close lock at 1731960567154Disabling compacts and flushes for region at 1731960567154Disabling writes for close at 1731960567154Writing region close event to WAL at 1731960567154Closed at 1731960567154 2024-11-18T20:09:27,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:09:27,157 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960567156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960567156"}]},"ts":"1731960567156"} 2024-11-18T20:09:27,160 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:09:27,161 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:09:27,162 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960567161"}]},"ts":"1731960567161"} 2024-11-18T20:09:27,163 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:09:27,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, ASSIGN}] 2024-11-18T20:09:27,165 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, ASSIGN 2024-11-18T20:09:27,166 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, ASSIGN; state=OFFLINE, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=false 2024-11-18T20:09:27,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=9c697d531cc5d270f339498b2c3c441b, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:27,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, ASSIGN because future has completed 2024-11-18T20:09:27,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:09:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:09:27,482 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,482 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:09:27,483 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,483 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:09:27,483 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,483 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,486 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,488 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c697d531cc5d270f339498b2c3c441b columnFamilyName A 2024-11-18T20:09:27,488 DEBUG [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:27,489 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] regionserver.HStore(327): Store=9c697d531cc5d270f339498b2c3c441b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:27,489 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,491 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c697d531cc5d270f339498b2c3c441b columnFamilyName B 2024-11-18T20:09:27,491 DEBUG [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:27,492 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] regionserver.HStore(327): Store=9c697d531cc5d270f339498b2c3c441b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:27,492 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,493 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c697d531cc5d270f339498b2c3c441b columnFamilyName C 2024-11-18T20:09:27,493 DEBUG [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:27,493 INFO [StoreOpener-9c697d531cc5d270f339498b2c3c441b-1 {}] regionserver.HStore(327): Store=9c697d531cc5d270f339498b2c3c441b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:27,493 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,494 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,494 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,495 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,495 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,495 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:09:27,496 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,498 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:09:27,499 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 9c697d531cc5d270f339498b2c3c441b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62542271, jitterRate=-0.06804753839969635}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:09:27,499 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:27,499 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 9c697d531cc5d270f339498b2c3c441b: Running coprocessor pre-open hook at 1731960567483Writing region info on filesystem at 1731960567483Initializing all the Stores at 1731960567485 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960567485Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960567485Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960567485Cleaning up temporary data from old regions at 1731960567495 (+10 ms)Running coprocessor post-open hooks at 1731960567499 (+4 ms)Region opened successfully at 1731960567499 2024-11-18T20:09:27,500 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., pid=59, masterSystemTime=1731960567476 2024-11-18T20:09:27,501 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,501 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:27,502 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=9c697d531cc5d270f339498b2c3c441b, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:27,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:09:27,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T20:09:27,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 in 182 msec 2024-11-18T20:09:27,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-18T20:09:27,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, ASSIGN in 342 msec 2024-11-18T20:09:27,509 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:09:27,509 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960567509"}]},"ts":"1731960567509"} 2024-11-18T20:09:27,511 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:09:27,511 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:09:27,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-18T20:09:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:09:28,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:28,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,501 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12bfe01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40619943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,505 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,508 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:09:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:09:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a5727ec 2024-11-18T20:09:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:09:28,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:09:28,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,513 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26fc9d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,514 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a08ea67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168add8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,518 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dc9059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,520 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa89662, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6f4481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,524 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39db15a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,525 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ee8f5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,527 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c25966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,529 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1068fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,531 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a362432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,532 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffa41e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4659b0ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,535 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8c95ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1929c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,538 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564f4474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbd01bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,541 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae43031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:28,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:28,543 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b8d564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:28,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:28,544 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:28,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4168ac77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:28,545 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,545 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,545 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,546 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,546 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,546 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:28,546 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,547 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-18T20:09:28,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,548 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,548 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:28,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,549 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,550 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:28,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:09:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:28,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:09:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:28,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:28,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:09:28,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/da05233e609741059005bce74deed6e3 is 50, key is test_row_0/A:col10/1731960568558/Put/seqid=0 2024-11-18T20:09:28,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960628584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960628585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960628588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960628589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960628590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742062_1238 (size=12001) 2024-11-18T20:09:28,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/da05233e609741059005bce74deed6e3 2024-11-18T20:09:28,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/5346356b671c43ab869fae4da71f947f is 50, key is test_row_0/B:col10/1731960568558/Put/seqid=0 2024-11-18T20:09:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742063_1239 (size=12001) 2024-11-18T20:09:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:09:28,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/5346356b671c43ab869fae4da71f947f 2024-11-18T20:09:28,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/9c5517f0873f416d83f971140a9053a6 is 50, key is test_row_0/C:col10/1731960568558/Put/seqid=0 2024-11-18T20:09:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742064_1240 (size=12001) 2024-11-18T20:09:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960628692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960628692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/9c5517f0873f416d83f971140a9053a6 2024-11-18T20:09:28,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/da05233e609741059005bce74deed6e3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/da05233e609741059005bce74deed6e3 2024-11-18T20:09:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960628702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:09:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960628702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960628703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:28,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/da05233e609741059005bce74deed6e3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:09:28,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/5346356b671c43ab869fae4da71f947f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/5346356b671c43ab869fae4da71f947f 2024-11-18T20:09:28,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/5346356b671c43ab869fae4da71f947f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:09:28,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/9c5517f0873f416d83f971140a9053a6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9c5517f0873f416d83f971140a9053a6 2024-11-18T20:09:28,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9c5517f0873f416d83f971140a9053a6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:09:28,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9c697d531cc5d270f339498b2c3c441b in 172ms, sequenceid=13, compaction requested=false 2024-11-18T20:09:28,733 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:09:28,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:28,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:09:28,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:28,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:09:28,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/5de3657e20464e6ea60042de1c606220 is 50, key is test_row_0/A:col10/1731960568588/Put/seqid=0 2024-11-18T20:09:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:09:28,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742065_1241 (size=12001) 2024-11-18T20:09:28,890 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/5de3657e20464e6ea60042de1c606220 2024-11-18T20:09:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/e0991fdfea4949a78ac30430be067a2c is 50, key is test_row_0/B:col10/1731960568588/Put/seqid=0 2024-11-18T20:09:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:28,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:28,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742066_1242 (size=12001) 2024-11-18T20:09:28,915 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/e0991fdfea4949a78ac30430be067a2c 2024-11-18T20:09:28,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960628913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960628913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960628915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960628917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960628918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/40258932ee2d4d0cb500c6b1ca1a2ae2 is 50, key is test_row_0/C:col10/1731960568588/Put/seqid=0 2024-11-18T20:09:28,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742067_1243 (size=12001) 2024-11-18T20:09:28,945 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/40258932ee2d4d0cb500c6b1ca1a2ae2 2024-11-18T20:09:28,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/5de3657e20464e6ea60042de1c606220 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5de3657e20464e6ea60042de1c606220 2024-11-18T20:09:28,955 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5de3657e20464e6ea60042de1c606220, entries=150, sequenceid=38, filesize=11.7 K 2024-11-18T20:09:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/e0991fdfea4949a78ac30430be067a2c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/e0991fdfea4949a78ac30430be067a2c 2024-11-18T20:09:28,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/e0991fdfea4949a78ac30430be067a2c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-18T20:09:28,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/40258932ee2d4d0cb500c6b1ca1a2ae2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/40258932ee2d4d0cb500c6b1ca1a2ae2 2024-11-18T20:09:28,967 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/40258932ee2d4d0cb500c6b1ca1a2ae2, entries=150, sequenceid=38, filesize=11.7 K 2024-11-18T20:09:28,967 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 9c697d531cc5d270f339498b2c3c441b in 108ms, sequenceid=38, compaction requested=false 2024-11-18T20:09:28,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:28,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:28,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-18T20:09:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-18T20:09:28,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T20:09:28,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 419 msec 2024-11-18T20:09:28,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 426 msec 2024-11-18T20:09:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:29,023 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:29,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:09:29,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/2ae5366352b84777a22a738b8a83cf6d is 50, key is test_row_0/A:col10/1731960569022/Put/seqid=0 2024-11-18T20:09:29,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742068_1244 (size=14341) 2024-11-18T20:09:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960629052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960629052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960629055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960629056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960629162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960629162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960629162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960629162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:09:29,182 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:29,182 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-18T20:09:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:09:29,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:29,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:29,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:29,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960629222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:09:29,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T20:09:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960629372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960629372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960629372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960629372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/2ae5366352b84777a22a738b8a83cf6d 2024-11-18T20:09:29,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/4d0da3a63c5a4278a43977d52b942415 is 50, key is test_row_0/B:col10/1731960569022/Put/seqid=0 2024-11-18T20:09:29,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742069_1245 (size=12001) 2024-11-18T20:09:29,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/4d0da3a63c5a4278a43977d52b942415 2024-11-18T20:09:29,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/556c9804f1d64555aa30b57c75984ef7 is 50, key is test_row_0/C:col10/1731960569022/Put/seqid=0 2024-11-18T20:09:29,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742070_1246 (size=12001) 2024-11-18T20:09:29,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T20:09:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:09:29,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T20:09:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960629682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960629682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960629682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960629682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960629731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:29,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T20:09:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:09:29,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/556c9804f1d64555aa30b57c75984ef7 2024-11-18T20:09:29,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/2ae5366352b84777a22a738b8a83cf6d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2ae5366352b84777a22a738b8a83cf6d 2024-11-18T20:09:29,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2ae5366352b84777a22a738b8a83cf6d, entries=200, sequenceid=51, filesize=14.0 K 2024-11-18T20:09:29,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/4d0da3a63c5a4278a43977d52b942415 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4d0da3a63c5a4278a43977d52b942415 2024-11-18T20:09:29,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4d0da3a63c5a4278a43977d52b942415, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:09:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/556c9804f1d64555aa30b57c75984ef7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/556c9804f1d64555aa30b57c75984ef7 2024-11-18T20:09:29,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/556c9804f1d64555aa30b57c75984ef7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:09:29,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9c697d531cc5d270f339498b2c3c441b in 861ms, sequenceid=51, compaction requested=true 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:29,884 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:29,884 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:29,885 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:29,885 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:29,885 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:29,885 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:29,885 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,885 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,886 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/da05233e609741059005bce74deed6e3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5de3657e20464e6ea60042de1c606220, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2ae5366352b84777a22a738b8a83cf6d] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=37.4 K 2024-11-18T20:09:29,886 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/5346356b671c43ab869fae4da71f947f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/e0991fdfea4949a78ac30430be067a2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4d0da3a63c5a4278a43977d52b942415] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.2 K 2024-11-18T20:09:29,886 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5346356b671c43ab869fae4da71f947f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731960568557 2024-11-18T20:09:29,886 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting da05233e609741059005bce74deed6e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731960568557 2024-11-18T20:09:29,886 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de3657e20464e6ea60042de1c606220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731960568581 2024-11-18T20:09:29,886 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e0991fdfea4949a78ac30430be067a2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731960568581 2024-11-18T20:09:29,887 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae5366352b84777a22a738b8a83cf6d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960568913 2024-11-18T20:09:29,887 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0da3a63c5a4278a43977d52b942415, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960568915 2024-11-18T20:09:29,900 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:29,900 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/4fd7cba07ada44679629ea488d9bab17 is 50, key is test_row_0/B:col10/1731960569022/Put/seqid=0 2024-11-18T20:09:29,902 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#205 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:29,902 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/85bc19f60c60402aa918880b45f5a38b is 50, key is test_row_0/A:col10/1731960569022/Put/seqid=0 2024-11-18T20:09:29,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742071_1247 (size=12104) 2024-11-18T20:09:29,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742072_1248 (size=12104) 2024-11-18T20:09:29,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T20:09:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:29,948 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:09:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9a868b4770fa4fdc9f2e3f905437aadd is 50, key is test_row_0/A:col10/1731960569053/Put/seqid=0 2024-11-18T20:09:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742073_1249 (size=12001) 2024-11-18T20:09:29,958 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9a868b4770fa4fdc9f2e3f905437aadd 2024-11-18T20:09:29,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/eeefd4e4a9fd4b028845a3329a11afd3 is 50, key is test_row_0/B:col10/1731960569053/Put/seqid=0 2024-11-18T20:09:29,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742074_1250 (size=12001) 2024-11-18T20:09:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:30,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:30,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960630201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960630201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960630203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960630205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960630311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960630312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960630312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960630312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,326 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/85bc19f60c60402aa918880b45f5a38b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/85bc19f60c60402aa918880b45f5a38b 2024-11-18T20:09:30,328 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/4fd7cba07ada44679629ea488d9bab17 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4fd7cba07ada44679629ea488d9bab17 2024-11-18T20:09:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:09:30,332 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 85bc19f60c60402aa918880b45f5a38b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:30,332 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 4fd7cba07ada44679629ea488d9bab17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:30,332 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:30,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:30,333 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960569884; duration=0sec 2024-11-18T20:09:30,333 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960569884; duration=0sec 2024-11-18T20:09:30,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:30,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:30,333 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:30,333 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:30,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:30,334 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:30,334 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:30,334 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:30,334 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9c5517f0873f416d83f971140a9053a6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/40258932ee2d4d0cb500c6b1ca1a2ae2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/556c9804f1d64555aa30b57c75984ef7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.2 K 2024-11-18T20:09:30,334 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5517f0873f416d83f971140a9053a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731960568557 2024-11-18T20:09:30,335 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 40258932ee2d4d0cb500c6b1ca1a2ae2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731960568581 2024-11-18T20:09:30,335 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 556c9804f1d64555aa30b57c75984ef7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960568915 2024-11-18T20:09:30,350 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:30,351 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c6921e29c3bc4d2b95d63e2ccec55fdf is 50, key is test_row_0/C:col10/1731960569022/Put/seqid=0 2024-11-18T20:09:30,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742075_1251 (size=12104) 2024-11-18T20:09:30,361 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c6921e29c3bc4d2b95d63e2ccec55fdf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c6921e29c3bc4d2b95d63e2ccec55fdf 2024-11-18T20:09:30,365 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into c6921e29c3bc4d2b95d63e2ccec55fdf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:30,365 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:30,365 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960569884; duration=0sec 2024-11-18T20:09:30,365 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:30,365 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:30,377 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/eeefd4e4a9fd4b028845a3329a11afd3 2024-11-18T20:09:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c790abf8e7bc4af2bec08b6b6caa7f96 is 50, key is test_row_0/C:col10/1731960569053/Put/seqid=0 2024-11-18T20:09:30,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742076_1252 (size=12001) 2024-11-18T20:09:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960630522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960630522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960630522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960630522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960630742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,744 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,789 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c790abf8e7bc4af2bec08b6b6caa7f96 2024-11-18T20:09:30,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9a868b4770fa4fdc9f2e3f905437aadd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9a868b4770fa4fdc9f2e3f905437aadd 2024-11-18T20:09:30,808 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9a868b4770fa4fdc9f2e3f905437aadd, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:09:30,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/eeefd4e4a9fd4b028845a3329a11afd3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eeefd4e4a9fd4b028845a3329a11afd3 2024-11-18T20:09:30,813 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eeefd4e4a9fd4b028845a3329a11afd3, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:09:30,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c790abf8e7bc4af2bec08b6b6caa7f96 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c790abf8e7bc4af2bec08b6b6caa7f96 2024-11-18T20:09:30,818 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c790abf8e7bc4af2bec08b6b6caa7f96, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:09:30,819 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9c697d531cc5d270f339498b2c3c441b in 872ms, sequenceid=74, compaction requested=false 2024-11-18T20:09:30,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:30,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:30,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-18T20:09:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-18T20:09:30,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-18T20:09:30,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6360 sec 2024-11-18T20:09:30,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6410 sec 2024-11-18T20:09:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:30,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:30,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:09:30,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/15ddc76a3e50497cac1b79791a02e694 is 50, key is test_row_0/A:col10/1731960570833/Put/seqid=0 2024-11-18T20:09:30,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742077_1253 (size=12001) 2024-11-18T20:09:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960630861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960630862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960630863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960630864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960630972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960630972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960630972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:30,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960630972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960631184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960631184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960631184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960631184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/15ddc76a3e50497cac1b79791a02e694 2024-11-18T20:09:31,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/c0c190f70427423788639e8f875a9bfc is 50, key is test_row_0/B:col10/1731960570833/Put/seqid=0 2024-11-18T20:09:31,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742078_1254 (size=12001) 2024-11-18T20:09:31,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/c0c190f70427423788639e8f875a9bfc 2024-11-18T20:09:31,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5095ade581264761b0224ac14fbbf09c is 50, key is test_row_0/C:col10/1731960570833/Put/seqid=0 2024-11-18T20:09:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:09:31,342 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:31,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-18T20:09:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:09:31,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:31,346 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:31,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:31,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742079_1255 (size=12001) 2024-11-18T20:09:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:09:31,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T20:09:31,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:31,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:31,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960631501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960631502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960631502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960631502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:31,552 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:09:31,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T20:09:31,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:31,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:31,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:09:31,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5095ade581264761b0224ac14fbbf09c 2024-11-18T20:09:31,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/15ddc76a3e50497cac1b79791a02e694 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/15ddc76a3e50497cac1b79791a02e694 2024-11-18T20:09:31,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/15ddc76a3e50497cac1b79791a02e694, entries=150, sequenceid=92, filesize=11.7 K 2024-11-18T20:09:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/c0c190f70427423788639e8f875a9bfc as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/c0c190f70427423788639e8f875a9bfc 2024-11-18T20:09:31,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/c0c190f70427423788639e8f875a9bfc, entries=150, sequenceid=92, filesize=11.7 K 2024-11-18T20:09:31,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5095ade581264761b0224ac14fbbf09c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5095ade581264761b0224ac14fbbf09c 2024-11-18T20:09:31,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5095ade581264761b0224ac14fbbf09c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-18T20:09:31,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 9c697d531cc5d270f339498b2c3c441b in 935ms, sequenceid=92, compaction requested=true 2024-11-18T20:09:31,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:31,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:31,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:31,769 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:31,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:31,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:31,770 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:31,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:31,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:31,771 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:31,771 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:31,771 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,771 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4fd7cba07ada44679629ea488d9bab17, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eeefd4e4a9fd4b028845a3329a11afd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/c0c190f70427423788639e8f875a9bfc] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.3 K 2024-11-18T20:09:31,771 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:31,771 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:31,771 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,771 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/85bc19f60c60402aa918880b45f5a38b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9a868b4770fa4fdc9f2e3f905437aadd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/15ddc76a3e50497cac1b79791a02e694] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.3 K 2024-11-18T20:09:31,772 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd7cba07ada44679629ea488d9bab17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960568915 2024-11-18T20:09:31,772 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bc19f60c60402aa918880b45f5a38b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960568915 2024-11-18T20:09:31,772 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting eeefd4e4a9fd4b028845a3329a11afd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731960569042 2024-11-18T20:09:31,772 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a868b4770fa4fdc9f2e3f905437aadd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731960569042 2024-11-18T20:09:31,772 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c190f70427423788639e8f875a9bfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960570200 2024-11-18T20:09:31,773 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15ddc76a3e50497cac1b79791a02e694, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960570200 2024-11-18T20:09:31,781 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:31,781 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/7a7ba619e8584c49ac01c0607812c5f3 is 50, key is test_row_0/A:col10/1731960570833/Put/seqid=0 2024-11-18T20:09:31,789 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#214 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:31,789 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/023d525377d3403abd8e28d52feb6422 is 50, key is test_row_0/B:col10/1731960570833/Put/seqid=0 2024-11-18T20:09:31,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742080_1256 (size=12207) 2024-11-18T20:09:31,800 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/7a7ba619e8584c49ac01c0607812c5f3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7a7ba619e8584c49ac01c0607812c5f3 2024-11-18T20:09:31,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T20:09:31,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,805 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:09:31,806 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 7a7ba619e8584c49ac01c0607812c5f3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:31,806 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:31,806 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960571769; duration=0sec 2024-11-18T20:09:31,806 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:31,806 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:31,806 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:31,808 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:31,808 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:31,808 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:31,808 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c6921e29c3bc4d2b95d63e2ccec55fdf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c790abf8e7bc4af2bec08b6b6caa7f96, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5095ade581264761b0224ac14fbbf09c] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.3 K 2024-11-18T20:09:31,808 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6921e29c3bc4d2b95d63e2ccec55fdf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960568915 2024-11-18T20:09:31,809 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c790abf8e7bc4af2bec08b6b6caa7f96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731960569042 2024-11-18T20:09:31,809 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5095ade581264761b0224ac14fbbf09c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960570200 2024-11-18T20:09:31,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742081_1257 (size=12207) 2024-11-18T20:09:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0946542cb8e240af86ac7e3db713553e is 50, key is test_row_0/A:col10/1731960570863/Put/seqid=0 2024-11-18T20:09:31,821 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:31,822 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/d03fe842eba64ffa8b7d748791e55e09 is 50, key is test_row_0/C:col10/1731960570833/Put/seqid=0 2024-11-18T20:09:31,823 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/023d525377d3403abd8e28d52feb6422 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/023d525377d3403abd8e28d52feb6422 2024-11-18T20:09:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742083_1259 (size=12207) 2024-11-18T20:09:31,836 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 023d525377d3403abd8e28d52feb6422(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:31,836 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:31,836 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960571769; duration=0sec 2024-11-18T20:09:31,837 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:31,837 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:31,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742082_1258 (size=12001) 2024-11-18T20:09:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:09:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:32,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:32,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960632023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960632024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960632024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960632025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960632131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960632131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960632132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960632132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,239 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/d03fe842eba64ffa8b7d748791e55e09 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/d03fe842eba64ffa8b7d748791e55e09 2024-11-18T20:09:32,243 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into d03fe842eba64ffa8b7d748791e55e09(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:32,243 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:32,243 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960571770; duration=0sec 2024-11-18T20:09:32,243 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:32,243 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:32,246 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0946542cb8e240af86ac7e3db713553e 2024-11-18T20:09:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/95a6fefc693448f5836b8c224a8c1592 is 50, key is test_row_0/B:col10/1731960570863/Put/seqid=0 2024-11-18T20:09:32,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742084_1260 (size=12001) 2024-11-18T20:09:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960632342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960632342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960632342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960632343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:09:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960632652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960632652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960632652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960632652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,658 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/95a6fefc693448f5836b8c224a8c1592 2024-11-18T20:09:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0d09895e4c794d97bab726b61c2c8fb0 is 50, key is test_row_0/C:col10/1731960570863/Put/seqid=0 2024-11-18T20:09:32,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742085_1261 (size=12001) 2024-11-18T20:09:32,677 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0d09895e4c794d97bab726b61c2c8fb0 2024-11-18T20:09:32,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0946542cb8e240af86ac7e3db713553e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0946542cb8e240af86ac7e3db713553e 2024-11-18T20:09:32,697 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0946542cb8e240af86ac7e3db713553e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-18T20:09:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/95a6fefc693448f5836b8c224a8c1592 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/95a6fefc693448f5836b8c224a8c1592 2024-11-18T20:09:32,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/95a6fefc693448f5836b8c224a8c1592, entries=150, sequenceid=113, filesize=11.7 K 2024-11-18T20:09:32,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0d09895e4c794d97bab726b61c2c8fb0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0d09895e4c794d97bab726b61c2c8fb0 2024-11-18T20:09:32,707 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0d09895e4c794d97bab726b61c2c8fb0, entries=150, sequenceid=113, filesize=11.7 K 2024-11-18T20:09:32,709 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 9c697d531cc5d270f339498b2c3c441b in 904ms, sequenceid=113, compaction requested=false 2024-11-18T20:09:32,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:32,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:32,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-18T20:09:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-18T20:09:32,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T20:09:32,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3640 sec 2024-11-18T20:09:32,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3710 sec 2024-11-18T20:09:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:32,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:32,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:09:32,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0c71baae370f4ed7875bd590d5432d5b is 50, key is test_row_0/A:col10/1731960572754/Put/seqid=0 2024-11-18T20:09:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742086_1262 (size=12101) 2024-11-18T20:09:32,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0c71baae370f4ed7875bd590d5432d5b 2024-11-18T20:09:32,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/679032947841480fbd51fc3a53b5539e is 50, key is test_row_0/B:col10/1731960572754/Put/seqid=0 2024-11-18T20:09:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742087_1263 (size=12101) 2024-11-18T20:09:32,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/679032947841480fbd51fc3a53b5539e 2024-11-18T20:09:32,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/63e2378a76a84d54a018b94269525c10 is 50, key is test_row_0/C:col10/1731960572754/Put/seqid=0 2024-11-18T20:09:32,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742088_1264 (size=12101) 2024-11-18T20:09:32,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960632827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:32,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960632942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960633151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960633162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960633162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960633162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960633162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/63e2378a76a84d54a018b94269525c10 2024-11-18T20:09:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0c71baae370f4ed7875bd590d5432d5b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0c71baae370f4ed7875bd590d5432d5b 2024-11-18T20:09:33,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0c71baae370f4ed7875bd590d5432d5b, entries=150, sequenceid=133, filesize=11.8 K 2024-11-18T20:09:33,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/679032947841480fbd51fc3a53b5539e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/679032947841480fbd51fc3a53b5539e 2024-11-18T20:09:33,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/679032947841480fbd51fc3a53b5539e, entries=150, sequenceid=133, filesize=11.8 K 2024-11-18T20:09:33,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/63e2378a76a84d54a018b94269525c10 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/63e2378a76a84d54a018b94269525c10 2024-11-18T20:09:33,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/63e2378a76a84d54a018b94269525c10, entries=150, sequenceid=133, filesize=11.8 K 2024-11-18T20:09:33,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9c697d531cc5d270f339498b2c3c441b in 460ms, sequenceid=133, compaction requested=true 2024-11-18T20:09:33,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:33,216 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:33,216 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:33,217 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:33,217 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:33,217 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,217 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7a7ba619e8584c49ac01c0607812c5f3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0946542cb8e240af86ac7e3db713553e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0c71baae370f4ed7875bd590d5432d5b] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.5 K 2024-11-18T20:09:33,217 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:33,217 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:33,217 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,217 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/023d525377d3403abd8e28d52feb6422, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/95a6fefc693448f5836b8c224a8c1592, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/679032947841480fbd51fc3a53b5539e] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.5 K 2024-11-18T20:09:33,218 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7ba619e8584c49ac01c0607812c5f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960570200 2024-11-18T20:09:33,218 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 023d525377d3403abd8e28d52feb6422, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960570200 2024-11-18T20:09:33,218 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0946542cb8e240af86ac7e3db713553e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731960570860 2024-11-18T20:09:33,218 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a6fefc693448f5836b8c224a8c1592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731960570860 2024-11-18T20:09:33,218 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c71baae370f4ed7875bd590d5432d5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960572022 2024-11-18T20:09:33,218 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 679032947841480fbd51fc3a53b5539e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960572022 2024-11-18T20:09:33,225 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#223 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:33,225 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:33,225 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/6928a4af540b433db34f573026c0948e is 50, key is test_row_0/B:col10/1731960572754/Put/seqid=0 2024-11-18T20:09:33,225 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/d326e73684a14848a7e21100c6b0130a is 50, key is test_row_0/A:col10/1731960572754/Put/seqid=0 2024-11-18T20:09:33,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742089_1265 (size=12409) 2024-11-18T20:09:33,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742090_1266 (size=12409) 2024-11-18T20:09:33,244 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/6928a4af540b433db34f573026c0948e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6928a4af540b433db34f573026c0948e 2024-11-18T20:09:33,249 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 6928a4af540b433db34f573026c0948e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:33,249 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:33,249 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960573216; duration=0sec 2024-11-18T20:09:33,249 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:33,249 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:33,249 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:33,250 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:33,250 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:33,250 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,251 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/d03fe842eba64ffa8b7d748791e55e09, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0d09895e4c794d97bab726b61c2c8fb0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/63e2378a76a84d54a018b94269525c10] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=35.5 K 2024-11-18T20:09:33,251 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d03fe842eba64ffa8b7d748791e55e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960570200 2024-11-18T20:09:33,251 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d09895e4c794d97bab726b61c2c8fb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731960570860 2024-11-18T20:09:33,251 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e2378a76a84d54a018b94269525c10, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960572022 2024-11-18T20:09:33,258 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:33,258 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/110a54661558488c87c483bc6621fe2c is 50, key is test_row_0/C:col10/1731960572754/Put/seqid=0 2024-11-18T20:09:33,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742091_1267 (size=12409) 2024-11-18T20:09:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:33,463 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:33,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:09:33,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/6e1e8290144f4b6c99634ed74e3db33a is 50, key is test_row_0/A:col10/1731960572818/Put/seqid=0 2024-11-18T20:09:33,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742092_1268 (size=14541) 2024-11-18T20:09:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:09:33,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:33,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-18T20:09:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:09:33,494 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:33,495 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:33,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960633512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:09:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960633622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,645 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/d326e73684a14848a7e21100c6b0130a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d326e73684a14848a7e21100c6b0130a 2024-11-18T20:09:33,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:09:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,649 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into d326e73684a14848a7e21100c6b0130a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:33,649 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:33,649 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960573215; duration=0sec 2024-11-18T20:09:33,649 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:33,649 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,666 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/110a54661558488c87c483bc6621fe2c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/110a54661558488c87c483bc6621fe2c 2024-11-18T20:09:33,670 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into 110a54661558488c87c483bc6621fe2c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:33,671 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:33,671 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960573216; duration=0sec 2024-11-18T20:09:33,671 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:33,671 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:33,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:09:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:09:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960633832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:33,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/6e1e8290144f4b6c99634ed74e3db33a 2024-11-18T20:09:33,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/dce3f5a8397b450ca80cd8b2a518e2bd is 50, key is test_row_0/B:col10/1731960572818/Put/seqid=0 2024-11-18T20:09:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742093_1269 (size=12151) 2024-11-18T20:09:33,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/dce3f5a8397b450ca80cd8b2a518e2bd 2024-11-18T20:09:33,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c is 50, key is test_row_0/C:col10/1731960572818/Put/seqid=0 2024-11-18T20:09:33,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742094_1270 (size=12151) 2024-11-18T20:09:33,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:09:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:33,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:09:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:34,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:09:34,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960634142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960634172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960634172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,174 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:34,174 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960634182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960634182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,184 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,185 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:09:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:34,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:34,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c 2024-11-18T20:09:34,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/6e1e8290144f4b6c99634ed74e3db33a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6e1e8290144f4b6c99634ed74e3db33a 2024-11-18T20:09:34,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6e1e8290144f4b6c99634ed74e3db33a, entries=200, sequenceid=154, filesize=14.2 K 2024-11-18T20:09:34,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/dce3f5a8397b450ca80cd8b2a518e2bd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/dce3f5a8397b450ca80cd8b2a518e2bd 2024-11-18T20:09:34,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/dce3f5a8397b450ca80cd8b2a518e2bd, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:09:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c 2024-11-18T20:09:34,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:09:34,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9c697d531cc5d270f339498b2c3c441b in 849ms, sequenceid=154, compaction requested=false 2024-11-18T20:09:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:34,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:09:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:34,412 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:09:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/2679a2443a1743048c638eb7e6a969eb is 50, key is test_row_0/A:col10/1731960573502/Put/seqid=0 2024-11-18T20:09:34,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742095_1271 (size=12151) 2024-11-18T20:09:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:09:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:34,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:34,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960634712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,821 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/2679a2443a1743048c638eb7e6a969eb 2024-11-18T20:09:34,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960634821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/abb0818965fb467da94e32e9c3108232 is 50, key is test_row_0/B:col10/1731960573502/Put/seqid=0 2024-11-18T20:09:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742096_1272 (size=12151) 2024-11-18T20:09:35,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960635032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:35,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/abb0818965fb467da94e32e9c3108232 2024-11-18T20:09:35,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/9a0c9260b68641bf814d62c340c7edf5 is 50, key is test_row_0/C:col10/1731960573502/Put/seqid=0 2024-11-18T20:09:35,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742097_1273 (size=12151) 2024-11-18T20:09:35,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/9a0c9260b68641bf814d62c340c7edf5 2024-11-18T20:09:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/2679a2443a1743048c638eb7e6a969eb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2679a2443a1743048c638eb7e6a969eb 2024-11-18T20:09:35,253 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2679a2443a1743048c638eb7e6a969eb, entries=150, sequenceid=172, filesize=11.9 K 2024-11-18T20:09:35,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/abb0818965fb467da94e32e9c3108232 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/abb0818965fb467da94e32e9c3108232 2024-11-18T20:09:35,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/abb0818965fb467da94e32e9c3108232, entries=150, sequenceid=172, filesize=11.9 K 2024-11-18T20:09:35,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/9a0c9260b68641bf814d62c340c7edf5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9a0c9260b68641bf814d62c340c7edf5 2024-11-18T20:09:35,262 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9a0c9260b68641bf814d62c340c7edf5, entries=150, sequenceid=172, filesize=11.9 K 2024-11-18T20:09:35,263 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9c697d531cc5d270f339498b2c3c441b in 851ms, sequenceid=172, compaction requested=true 2024-11-18T20:09:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-18T20:09:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-18T20:09:35,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-18T20:09:35,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-11-18T20:09:35,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7750 sec 2024-11-18T20:09:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:35,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:35,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:09:35,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0549f190ff114241b4e069e9a0dfb58e is 50, key is test_row_0/A:col10/1731960575343/Put/seqid=0 2024-11-18T20:09:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742098_1274 (size=14541) 2024-11-18T20:09:35,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0549f190ff114241b4e069e9a0dfb58e 2024-11-18T20:09:35,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/97037468176743d1b945b640b36e920d is 50, key is test_row_0/B:col10/1731960575343/Put/seqid=0 2024-11-18T20:09:35,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742099_1275 (size=12151) 2024-11-18T20:09:35,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960635394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:35,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960635502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:09:35,642 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:35,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-18T20:09:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:09:35,644 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:35,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:35,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960635712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:09:35,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/97037468176743d1b945b640b36e920d 2024-11-18T20:09:35,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/6ac9fea7799b416f92dfcdfc650081d7 is 50, key is test_row_0/C:col10/1731960575343/Put/seqid=0 2024-11-18T20:09:35,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742100_1276 (size=12151) 2024-11-18T20:09:35,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/6ac9fea7799b416f92dfcdfc650081d7 2024-11-18T20:09:35,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0549f190ff114241b4e069e9a0dfb58e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0549f190ff114241b4e069e9a0dfb58e 2024-11-18T20:09:35,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0549f190ff114241b4e069e9a0dfb58e, entries=200, sequenceid=193, filesize=14.2 K 2024-11-18T20:09:35,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/97037468176743d1b945b640b36e920d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97037468176743d1b945b640b36e920d 2024-11-18T20:09:35,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97037468176743d1b945b640b36e920d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-18T20:09:35,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/6ac9fea7799b416f92dfcdfc650081d7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/6ac9fea7799b416f92dfcdfc650081d7 2024-11-18T20:09:35,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/6ac9fea7799b416f92dfcdfc650081d7, entries=150, sequenceid=193, filesize=11.9 K 2024-11-18T20:09:35,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9c697d531cc5d270f339498b2c3c441b in 449ms, sequenceid=193, compaction requested=true 2024-11-18T20:09:35,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:35,795 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:35,795 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:35,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:35,796 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:35,796 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:35,796 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:35,796 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:35,796 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:35,796 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:35,796 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6928a4af540b433db34f573026c0948e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/dce3f5a8397b450ca80cd8b2a518e2bd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/abb0818965fb467da94e32e9c3108232, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97037468176743d1b945b640b36e920d] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=47.7 K 2024-11-18T20:09:35,796 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d326e73684a14848a7e21100c6b0130a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6e1e8290144f4b6c99634ed74e3db33a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2679a2443a1743048c638eb7e6a969eb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0549f190ff114241b4e069e9a0dfb58e] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=52.4 K 2024-11-18T20:09:35,796 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d326e73684a14848a7e21100c6b0130a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960572022 2024-11-18T20:09:35,796 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6928a4af540b433db34f573026c0948e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960572022 2024-11-18T20:09:35,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:09:35,797 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1e8290144f4b6c99634ed74e3db33a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731960572818 2024-11-18T20:09:35,797 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting dce3f5a8397b450ca80cd8b2a518e2bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731960572818 2024-11-18T20:09:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:35,797 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:09:35,797 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2679a2443a1743048c638eb7e6a969eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731960573497 2024-11-18T20:09:35,797 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting abb0818965fb467da94e32e9c3108232, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731960573497 2024-11-18T20:09:35,797 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0549f190ff114241b4e069e9a0dfb58e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960574689 2024-11-18T20:09:35,797 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 97037468176743d1b945b640b36e920d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960574689 2024-11-18T20:09:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9174fbb108b14e9c8d94dbf9a8f0308e is 50, key is test_row_0/A:col10/1731960575392/Put/seqid=0 2024-11-18T20:09:35,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742101_1277 (size=12151) 2024-11-18T20:09:35,811 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:35,811 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#235 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:35,811 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/9fe4934ddfd04a8e957d2a694989e707 is 50, key is test_row_0/B:col10/1731960575343/Put/seqid=0 2024-11-18T20:09:35,811 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/845a66e31b1a45b391be796add1e0361 is 50, key is test_row_0/A:col10/1731960575343/Put/seqid=0 2024-11-18T20:09:35,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742103_1279 (size=12595) 2024-11-18T20:09:35,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742102_1278 (size=12595) 2024-11-18T20:09:35,822 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/9fe4934ddfd04a8e957d2a694989e707 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/9fe4934ddfd04a8e957d2a694989e707 2024-11-18T20:09:35,828 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 9fe4934ddfd04a8e957d2a694989e707(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:35,828 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:35,828 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=12, startTime=1731960575795; duration=0sec 2024-11-18T20:09:35,828 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:35,828 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:35,828 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:35,830 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:35,830 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:35,830 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:35,831 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/110a54661558488c87c483bc6621fe2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9a0c9260b68641bf814d62c340c7edf5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/6ac9fea7799b416f92dfcdfc650081d7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=47.7 K 2024-11-18T20:09:35,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 110a54661558488c87c483bc6621fe2c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960572022 2024-11-18T20:09:35,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ab2bf8a9014dc2ab2c4b3833ee9f6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731960572818 2024-11-18T20:09:35,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0c9260b68641bf814d62c340c7edf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731960573497 2024-11-18T20:09:35,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac9fea7799b416f92dfcdfc650081d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960574689 2024-11-18T20:09:35,840 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:35,840 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/171fece8dc934f16a1fc903cf2014e3f is 50, key is test_row_0/C:col10/1731960575343/Put/seqid=0 2024-11-18T20:09:35,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742104_1280 (size=12595) 2024-11-18T20:09:35,849 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/171fece8dc934f16a1fc903cf2014e3f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/171fece8dc934f16a1fc903cf2014e3f 2024-11-18T20:09:35,854 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into 171fece8dc934f16a1fc903cf2014e3f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:35,854 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:35,854 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=12, startTime=1731960575795; duration=0sec 2024-11-18T20:09:35,854 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:35,854 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:09:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960636095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960636192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960636192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,195 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:36,195 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960636202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960636202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,205 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,207 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9174fbb108b14e9c8d94dbf9a8f0308e 2024-11-18T20:09:36,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960636212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,214 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/eba95a7ee251410da57749d7ab6e5013 is 50, key is test_row_0/B:col10/1731960575392/Put/seqid=0 2024-11-18T20:09:36,221 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/845a66e31b1a45b391be796add1e0361 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/845a66e31b1a45b391be796add1e0361 2024-11-18T20:09:36,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742105_1281 (size=12151) 2024-11-18T20:09:36,223 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/eba95a7ee251410da57749d7ab6e5013 2024-11-18T20:09:36,228 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 845a66e31b1a45b391be796add1e0361(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:36,228 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:36,228 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=12, startTime=1731960575795; duration=0sec 2024-11-18T20:09:36,228 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:36,228 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/85052b62adb448fba0ca5af06688575d is 50, key is test_row_0/C:col10/1731960575392/Put/seqid=0 2024-11-18T20:09:36,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742106_1282 (size=12151) 2024-11-18T20:09:36,240 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/85052b62adb448fba0ca5af06688575d 2024-11-18T20:09:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9174fbb108b14e9c8d94dbf9a8f0308e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9174fbb108b14e9c8d94dbf9a8f0308e 2024-11-18T20:09:36,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9174fbb108b14e9c8d94dbf9a8f0308e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T20:09:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/eba95a7ee251410da57749d7ab6e5013 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eba95a7ee251410da57749d7ab6e5013 2024-11-18T20:09:36,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eba95a7ee251410da57749d7ab6e5013, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T20:09:36,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/85052b62adb448fba0ca5af06688575d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/85052b62adb448fba0ca5af06688575d 2024-11-18T20:09:36,263 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/85052b62adb448fba0ca5af06688575d, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T20:09:36,264 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 9c697d531cc5d270f339498b2c3c441b in 467ms, sequenceid=208, compaction requested=false 2024-11-18T20:09:36,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:36,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-18T20:09:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-18T20:09:36,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T20:09:36,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 621 msec 2024-11-18T20:09:36,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 627 msec 2024-11-18T20:09:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:09:36,272 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:36,272 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-18T20:09:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:36,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:36,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:36,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:36,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:36,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:09:36,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/34c67d4220c247c2a659a90949af0da3 is 50, key is test_row_0/A:col10/1731960576078/Put/seqid=0 2024-11-18T20:09:36,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742107_1283 (size=14541) 2024-11-18T20:09:36,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:36,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:36,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960636452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960636562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:36,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:36,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:36,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960636772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:36,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/34c67d4220c247c2a659a90949af0da3 2024-11-18T20:09:36,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/97b126ce02d74297a54a7a65abd8480d is 50, key is test_row_0/B:col10/1731960576078/Put/seqid=0 2024-11-18T20:09:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742108_1284 (size=12151) 2024-11-18T20:09:36,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:36,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:37,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960637082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:37,095 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:09:37,095 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:09:37,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/97b126ce02d74297a54a7a65abd8480d 2024-11-18T20:09:37,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5ed717fb78ad4e088532dea956146dcf is 50, key is test_row_0/C:col10/1731960576078/Put/seqid=0 2024-11-18T20:09:37,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742109_1285 (size=12151) 2024-11-18T20:09:37,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:37,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960637592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:37,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5ed717fb78ad4e088532dea956146dcf 2024-11-18T20:09:37,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/34c67d4220c247c2a659a90949af0da3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/34c67d4220c247c2a659a90949af0da3 2024-11-18T20:09:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:37,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/34c67d4220c247c2a659a90949af0da3, entries=200, sequenceid=233, filesize=14.2 K 2024-11-18T20:09:37,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/97b126ce02d74297a54a7a65abd8480d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97b126ce02d74297a54a7a65abd8480d 2024-11-18T20:09:37,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97b126ce02d74297a54a7a65abd8480d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-18T20:09:37,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5ed717fb78ad4e088532dea956146dcf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed717fb78ad4e088532dea956146dcf 2024-11-18T20:09:37,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed717fb78ad4e088532dea956146dcf, entries=150, sequenceid=233, filesize=11.9 K 2024-11-18T20:09:37,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9c697d531cc5d270f339498b2c3c441b in 1249ms, sequenceid=233, compaction requested=true 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:37,663 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:37,663 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:37,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:37,664 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:37,664 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:37,664 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:37,664 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:37,664 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,664 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,665 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/845a66e31b1a45b391be796add1e0361, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9174fbb108b14e9c8d94dbf9a8f0308e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/34c67d4220c247c2a659a90949af0da3] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=38.4 K 2024-11-18T20:09:37,665 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/9fe4934ddfd04a8e957d2a694989e707, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eba95a7ee251410da57749d7ab6e5013, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97b126ce02d74297a54a7a65abd8480d] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.0 K 2024-11-18T20:09:37,665 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe4934ddfd04a8e957d2a694989e707, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960574689 2024-11-18T20:09:37,665 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 845a66e31b1a45b391be796add1e0361, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960574689 2024-11-18T20:09:37,665 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting eba95a7ee251410da57749d7ab6e5013, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731960575380 2024-11-18T20:09:37,665 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9174fbb108b14e9c8d94dbf9a8f0308e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731960575380 2024-11-18T20:09:37,665 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b126ce02d74297a54a7a65abd8480d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960576078 2024-11-18T20:09:37,665 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c67d4220c247c2a659a90949af0da3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960576063 2024-11-18T20:09:37,672 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:37,672 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f691ff1c6394452bbfc10f7057ad5c51 is 50, key is test_row_0/A:col10/1731960576078/Put/seqid=0 2024-11-18T20:09:37,672 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:37,673 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3a2d9a7925ee4250ac91ac1e46e64558 is 50, key is test_row_0/B:col10/1731960576078/Put/seqid=0 2024-11-18T20:09:37,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742110_1286 (size=12697) 2024-11-18T20:09:37,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742111_1287 (size=12697) 2024-11-18T20:09:37,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:09:37,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:37,802 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:09:37,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/1c978dcc9de74c4b9c93683f807424d1 is 50, key is test_row_0/A:col10/1731960576447/Put/seqid=0 2024-11-18T20:09:37,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742112_1288 (size=12151) 2024-11-18T20:09:38,085 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f691ff1c6394452bbfc10f7057ad5c51 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f691ff1c6394452bbfc10f7057ad5c51 2024-11-18T20:09:38,089 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3a2d9a7925ee4250ac91ac1e46e64558 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3a2d9a7925ee4250ac91ac1e46e64558 2024-11-18T20:09:38,089 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into f691ff1c6394452bbfc10f7057ad5c51(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:38,089 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:38,089 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960577663; duration=0sec 2024-11-18T20:09:38,090 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:38,090 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:38,090 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:38,091 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:38,091 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:38,091 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:38,091 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/171fece8dc934f16a1fc903cf2014e3f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/85052b62adb448fba0ca5af06688575d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed717fb78ad4e088532dea956146dcf] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.0 K 2024-11-18T20:09:38,093 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 171fece8dc934f16a1fc903cf2014e3f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960574689 2024-11-18T20:09:38,094 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85052b62adb448fba0ca5af06688575d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731960575380 2024-11-18T20:09:38,094 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed717fb78ad4e088532dea956146dcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960576078 2024-11-18T20:09:38,096 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 3a2d9a7925ee4250ac91ac1e46e64558(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:38,096 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:38,096 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960577663; duration=0sec 2024-11-18T20:09:38,096 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:38,096 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:38,103 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#246 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:38,103 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/267ccf8cde7d472eb101cecd7d357836 is 50, key is test_row_0/C:col10/1731960576078/Put/seqid=0 2024-11-18T20:09:38,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742113_1289 (size=12697) 2024-11-18T20:09:38,210 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/1c978dcc9de74c4b9c93683f807424d1 2024-11-18T20:09:38,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/19c63659074047278896b5550ff97ad5 is 50, key is test_row_0/B:col10/1731960576447/Put/seqid=0 2024-11-18T20:09:38,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742114_1290 (size=12151) 2024-11-18T20:09:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:38,512 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/267ccf8cde7d472eb101cecd7d357836 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/267ccf8cde7d472eb101cecd7d357836 2024-11-18T20:09:38,516 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into 267ccf8cde7d472eb101cecd7d357836(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:38,516 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:38,516 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960577663; duration=0sec 2024-11-18T20:09:38,516 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:38,516 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:38,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:38,622 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/19c63659074047278896b5550ff97ad5 2024-11-18T20:09:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/12303603965646b4b1c0ddf37b22fd0f is 50, key is test_row_0/C:col10/1731960576447/Put/seqid=0 2024-11-18T20:09:38,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742115_1291 (size=12151) 2024-11-18T20:09:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960638676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:38,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960638781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:38,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960638992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:39,033 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/12303603965646b4b1c0ddf37b22fd0f 2024-11-18T20:09:39,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/1c978dcc9de74c4b9c93683f807424d1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/1c978dcc9de74c4b9c93683f807424d1 2024-11-18T20:09:39,040 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/1c978dcc9de74c4b9c93683f807424d1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T20:09:39,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/19c63659074047278896b5550ff97ad5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/19c63659074047278896b5550ff97ad5 2024-11-18T20:09:39,045 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/19c63659074047278896b5550ff97ad5, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T20:09:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/12303603965646b4b1c0ddf37b22fd0f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/12303603965646b4b1c0ddf37b22fd0f 2024-11-18T20:09:39,050 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/12303603965646b4b1c0ddf37b22fd0f, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T20:09:39,050 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9c697d531cc5d270f339498b2c3c441b in 1248ms, sequenceid=247, compaction requested=false 2024-11-18T20:09:39,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-18T20:09:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-18T20:09:39,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T20:09:39,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7770 sec 2024-11-18T20:09:39,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.7810 sec 2024-11-18T20:09:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:39,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:39,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:09:39,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/86c7a08b95094378b24cf0929d82f517 is 50, key is test_row_0/A:col10/1731960578670/Put/seqid=0 2024-11-18T20:09:39,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742116_1292 (size=14741) 2024-11-18T20:09:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960639347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:39,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960639461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:39,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960639672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:39,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/86c7a08b95094378b24cf0929d82f517 2024-11-18T20:09:39,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/755b2c36f490443f92fb809b81e8fcb8 is 50, key is test_row_0/B:col10/1731960578670/Put/seqid=0 2024-11-18T20:09:39,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742117_1293 (size=12301) 2024-11-18T20:09:39,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960639982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:40,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/755b2c36f490443f92fb809b81e8fcb8 2024-11-18T20:09:40,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5ed46c0c42c3467ab0e37b4fabdc1d05 is 50, key is test_row_0/C:col10/1731960578670/Put/seqid=0 2024-11-18T20:09:40,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742118_1294 (size=12301) 2024-11-18T20:09:40,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42084 deadline: 1731960640212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:40,215 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42058 deadline: 1731960640222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42030 deadline: 1731960640222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:40,226 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:40,227 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:40,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42052 deadline: 1731960640252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:40,254 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in 9c697d531cc5d270f339498b2c3c441b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:09:40,432 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:40,433 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-18T20:09:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:09:40,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:40,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:40,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:40,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960640492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:40,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5ed46c0c42c3467ab0e37b4fabdc1d05 2024-11-18T20:09:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:09:40,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/86c7a08b95094378b24cf0929d82f517 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/86c7a08b95094378b24cf0929d82f517 2024-11-18T20:09:40,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/86c7a08b95094378b24cf0929d82f517, entries=200, sequenceid=273, filesize=14.4 K 2024-11-18T20:09:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-18T20:09:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/755b2c36f490443f92fb809b81e8fcb8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/755b2c36f490443f92fb809b81e8fcb8 2024-11-18T20:09:40,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/755b2c36f490443f92fb809b81e8fcb8, entries=150, sequenceid=273, filesize=12.0 K 2024-11-18T20:09:40,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/5ed46c0c42c3467ab0e37b4fabdc1d05 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed46c0c42c3467ab0e37b4fabdc1d05 2024-11-18T20:09:40,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed46c0c42c3467ab0e37b4fabdc1d05, entries=150, sequenceid=273, filesize=12.0 K 2024-11-18T20:09:40,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9c697d531cc5d270f339498b2c3c441b in 1253ms, sequenceid=273, compaction requested=true 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:40,558 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:40,558 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:40,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:40,559 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:40,559 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:40,559 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:40,559 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:40,559 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:40,559 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:40,559 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f691ff1c6394452bbfc10f7057ad5c51, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/1c978dcc9de74c4b9c93683f807424d1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/86c7a08b95094378b24cf0929d82f517] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=38.7 K 2024-11-18T20:09:40,559 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3a2d9a7925ee4250ac91ac1e46e64558, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/19c63659074047278896b5550ff97ad5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/755b2c36f490443f92fb809b81e8fcb8] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.3 K 2024-11-18T20:09:40,560 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f691ff1c6394452bbfc10f7057ad5c51, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960576078 2024-11-18T20:09:40,560 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a2d9a7925ee4250ac91ac1e46e64558, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960576078 2024-11-18T20:09:40,560 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c63659074047278896b5550ff97ad5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731960576443 2024-11-18T20:09:40,560 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c978dcc9de74c4b9c93683f807424d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731960576443 2024-11-18T20:09:40,561 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86c7a08b95094378b24cf0929d82f517, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731960578659 2024-11-18T20:09:40,561 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 755b2c36f490443f92fb809b81e8fcb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731960578670 2024-11-18T20:09:40,567 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#252 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:40,567 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/25805e4ccd3c441d94bcab3d269bfeb6 is 50, key is test_row_0/B:col10/1731960578670/Put/seqid=0 2024-11-18T20:09:40,567 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:40,568 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f16decf02fc24a968b5d806e005663b5 is 50, key is test_row_0/A:col10/1731960578670/Put/seqid=0 2024-11-18T20:09:40,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742119_1295 (size=12949) 2024-11-18T20:09:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742120_1296 (size=12949) 2024-11-18T20:09:40,586 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f16decf02fc24a968b5d806e005663b5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f16decf02fc24a968b5d806e005663b5 2024-11-18T20:09:40,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T20:09:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:40,588 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:09:40,591 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into f16decf02fc24a968b5d806e005663b5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:40,592 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:40,592 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960580558; duration=0sec 2024-11-18T20:09:40,592 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:40,592 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:40,592 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/371d960060084c51851f23a933bfa674 is 50, key is test_row_0/A:col10/1731960579327/Put/seqid=0 2024-11-18T20:09:40,593 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:40,593 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:40,593 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:40,593 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/267ccf8cde7d472eb101cecd7d357836, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/12303603965646b4b1c0ddf37b22fd0f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed46c0c42c3467ab0e37b4fabdc1d05] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.3 K 2024-11-18T20:09:40,593 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267ccf8cde7d472eb101cecd7d357836, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960576078 2024-11-18T20:09:40,593 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12303603965646b4b1c0ddf37b22fd0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731960576443 2024-11-18T20:09:40,594 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed46c0c42c3467ab0e37b4fabdc1d05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731960578670 2024-11-18T20:09:40,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742121_1297 (size=12301) 2024-11-18T20:09:40,596 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/371d960060084c51851f23a933bfa674 2024-11-18T20:09:40,604 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:40,605 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/8ca2d7a3fa594bdeab718374ad1684ba is 50, key is test_row_0/C:col10/1731960578670/Put/seqid=0 2024-11-18T20:09:40,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/35f26f62d86f45a9aa6ed6d2074babaa is 50, key is test_row_0/B:col10/1731960579327/Put/seqid=0 2024-11-18T20:09:40,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742122_1298 (size=12949) 2024-11-18T20:09:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742123_1299 (size=12301) 2024-11-18T20:09:40,624 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/35f26f62d86f45a9aa6ed6d2074babaa 2024-11-18T20:09:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/95959906818a4039a20d2db9a39b825f is 50, key is test_row_0/C:col10/1731960579327/Put/seqid=0 2024-11-18T20:09:40,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742124_1300 (size=12301) 2024-11-18T20:09:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:09:40,975 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/25805e4ccd3c441d94bcab3d269bfeb6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/25805e4ccd3c441d94bcab3d269bfeb6 2024-11-18T20:09:40,979 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 25805e4ccd3c441d94bcab3d269bfeb6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:40,979 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:40,979 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960580558; duration=0sec 2024-11-18T20:09:40,979 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:40,979 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:09:41,065 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/95959906818a4039a20d2db9a39b825f 2024-11-18T20:09:41,068 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/8ca2d7a3fa594bdeab718374ad1684ba as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/8ca2d7a3fa594bdeab718374ad1684ba 2024-11-18T20:09:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/371d960060084c51851f23a933bfa674 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/371d960060084c51851f23a933bfa674 2024-11-18T20:09:41,073 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into 8ca2d7a3fa594bdeab718374ad1684ba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:41,074 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:41,074 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960580558; duration=0sec 2024-11-18T20:09:41,074 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:41,074 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:41,074 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/371d960060084c51851f23a933bfa674, entries=150, sequenceid=286, filesize=12.0 K 2024-11-18T20:09:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/35f26f62d86f45a9aa6ed6d2074babaa as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/35f26f62d86f45a9aa6ed6d2074babaa 2024-11-18T20:09:41,080 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/35f26f62d86f45a9aa6ed6d2074babaa, entries=150, sequenceid=286, filesize=12.0 K 2024-11-18T20:09:41,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/95959906818a4039a20d2db9a39b825f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/95959906818a4039a20d2db9a39b825f 2024-11-18T20:09:41,093 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/95959906818a4039a20d2db9a39b825f, entries=150, sequenceid=286, filesize=12.0 K 2024-11-18T20:09:41,095 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 9c697d531cc5d270f339498b2c3c441b in 507ms, sequenceid=286, compaction requested=false 2024-11-18T20:09:41,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:41,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:41,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-18T20:09:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-18T20:09:41,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T20:09:41,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 662 msec 2024-11-18T20:09:41,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 678 msec 2024-11-18T20:09:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:41,520 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:41,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:09:41,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f56a38f49a4046dfa9053f04ea8dd8ca is 50, key is test_row_0/A:col10/1731960581517/Put/seqid=0 2024-11-18T20:09:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742125_1301 (size=14741) 2024-11-18T20:09:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:09:41,576 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:41,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-18T20:09:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:41,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:41,581 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:41,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960641611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:41,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960641722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:41,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:41,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:41,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:41,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960641932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:41,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f56a38f49a4046dfa9053f04ea8dd8ca 2024-11-18T20:09:41,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/ed05d8193736492cbefe1686e79e6bc0 is 50, key is test_row_0/B:col10/1731960581517/Put/seqid=0 2024-11-18T20:09:42,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742126_1302 (size=12301) 2024-11-18T20:09:42,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/ed05d8193736492cbefe1686e79e6bc0 2024-11-18T20:09:42,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0ed7bd95660f42a8a7659077ca204a25 is 50, key is test_row_0/C:col10/1731960581517/Put/seqid=0 2024-11-18T20:09:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742127_1303 (size=12301) 2024-11-18T20:09:42,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960642241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:42,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:42,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:42,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:42,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0ed7bd95660f42a8a7659077ca204a25 2024-11-18T20:09:42,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/f56a38f49a4046dfa9053f04ea8dd8ca as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f56a38f49a4046dfa9053f04ea8dd8ca 2024-11-18T20:09:42,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f56a38f49a4046dfa9053f04ea8dd8ca, entries=200, sequenceid=300, filesize=14.4 K 2024-11-18T20:09:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/ed05d8193736492cbefe1686e79e6bc0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ed05d8193736492cbefe1686e79e6bc0 2024-11-18T20:09:42,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ed05d8193736492cbefe1686e79e6bc0, entries=150, sequenceid=300, filesize=12.0 K 2024-11-18T20:09:42,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0ed7bd95660f42a8a7659077ca204a25 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0ed7bd95660f42a8a7659077ca204a25 2024-11-18T20:09:42,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0ed7bd95660f42a8a7659077ca204a25, entries=150, sequenceid=300, filesize=12.0 K 2024-11-18T20:09:42,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9c697d531cc5d270f339498b2c3c441b in 1076ms, sequenceid=300, compaction requested=true 2024-11-18T20:09:42,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:42,596 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:42,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:42,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:42,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:42,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:42,597 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:42,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:42,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:42,598 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:42,598 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:42,598 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,598 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/25805e4ccd3c441d94bcab3d269bfeb6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/35f26f62d86f45a9aa6ed6d2074babaa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ed05d8193736492cbefe1686e79e6bc0] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.7 K 2024-11-18T20:09:42,598 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:42,598 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:42,598 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,598 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f16decf02fc24a968b5d806e005663b5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/371d960060084c51851f23a933bfa674, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f56a38f49a4046dfa9053f04ea8dd8ca] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=39.1 K 2024-11-18T20:09:42,603 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16decf02fc24a968b5d806e005663b5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731960578670 2024-11-18T20:09:42,603 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 25805e4ccd3c441d94bcab3d269bfeb6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731960578670 2024-11-18T20:09:42,604 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371d960060084c51851f23a933bfa674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731960579327 2024-11-18T20:09:42,604 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f26f62d86f45a9aa6ed6d2074babaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731960579327 2024-11-18T20:09:42,604 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56a38f49a4046dfa9053f04ea8dd8ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731960581511 2024-11-18T20:09:42,604 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ed05d8193736492cbefe1686e79e6bc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731960581513 2024-11-18T20:09:42,613 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:42,614 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:42,614 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/da3b0c9ab4c84d13b58dd096914b3185 is 50, key is test_row_0/B:col10/1731960581517/Put/seqid=0 2024-11-18T20:09:42,614 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9d2ae96514074af5a94a92dd77751519 is 50, key is test_row_0/A:col10/1731960581517/Put/seqid=0 2024-11-18T20:09:42,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742128_1304 (size=13051) 2024-11-18T20:09:42,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742129_1305 (size=13051) 2024-11-18T20:09:42,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:09:42,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,656 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:09:42,657 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/9d2ae96514074af5a94a92dd77751519 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9d2ae96514074af5a94a92dd77751519 2024-11-18T20:09:42,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/12e486bc59cc49e09df5742483ab6079 is 50, key is test_row_0/A:col10/1731960581601/Put/seqid=0 2024-11-18T20:09:42,664 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 9d2ae96514074af5a94a92dd77751519(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:42,664 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:42,664 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960582596; duration=0sec 2024-11-18T20:09:42,664 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:42,664 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:42,664 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:42,666 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:42,666 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:42,666 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:42,666 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/8ca2d7a3fa594bdeab718374ad1684ba, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/95959906818a4039a20d2db9a39b825f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0ed7bd95660f42a8a7659077ca204a25] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.7 K 2024-11-18T20:09:42,666 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ca2d7a3fa594bdeab718374ad1684ba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731960578670 2024-11-18T20:09:42,667 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95959906818a4039a20d2db9a39b825f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731960579327 2024-11-18T20:09:42,667 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed7bd95660f42a8a7659077ca204a25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731960581513 2024-11-18T20:09:42,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742130_1306 (size=12301) 2024-11-18T20:09:42,685 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:42,685 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/513a98ac0b954e20bba45f4158b6a144 is 50, key is test_row_0/C:col10/1731960581517/Put/seqid=0 2024-11-18T20:09:42,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742131_1307 (size=13051) 2024-11-18T20:09:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:42,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:42,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960642781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:42,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960642891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:43,031 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/da3b0c9ab4c84d13b58dd096914b3185 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/da3b0c9ab4c84d13b58dd096914b3185 2024-11-18T20:09:43,035 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into da3b0c9ab4c84d13b58dd096914b3185(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:43,036 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:43,036 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960582596; duration=0sec 2024-11-18T20:09:43,036 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:43,036 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:43,082 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/12e486bc59cc49e09df5742483ab6079 2024-11-18T20:09:43,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/42f46c7a71cd4896a47841dd99d2f1aa is 50, key is test_row_0/B:col10/1731960581601/Put/seqid=0 2024-11-18T20:09:43,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742132_1308 (size=12301) 2024-11-18T20:09:43,098 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/513a98ac0b954e20bba45f4158b6a144 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/513a98ac0b954e20bba45f4158b6a144 2024-11-18T20:09:43,103 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into 513a98ac0b954e20bba45f4158b6a144(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:43,103 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:43,103 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960582597; duration=0sec 2024-11-18T20:09:43,103 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:43,103 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:43,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960643102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:43,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960643411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:43,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/42f46c7a71cd4896a47841dd99d2f1aa 2024-11-18T20:09:43,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/121a51d7e981485ab11bb08efa244c33 is 50, key is test_row_0/C:col10/1731960581601/Put/seqid=0 2024-11-18T20:09:43,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742133_1309 (size=12301) 2024-11-18T20:09:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:43,913 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/121a51d7e981485ab11bb08efa244c33 2024-11-18T20:09:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/12e486bc59cc49e09df5742483ab6079 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/12e486bc59cc49e09df5742483ab6079 2024-11-18T20:09:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960643922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:43,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/12e486bc59cc49e09df5742483ab6079, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T20:09:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/42f46c7a71cd4896a47841dd99d2f1aa as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/42f46c7a71cd4896a47841dd99d2f1aa 2024-11-18T20:09:43,936 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/42f46c7a71cd4896a47841dd99d2f1aa, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T20:09:43,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/121a51d7e981485ab11bb08efa244c33 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/121a51d7e981485ab11bb08efa244c33 2024-11-18T20:09:43,945 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/121a51d7e981485ab11bb08efa244c33, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T20:09:43,947 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9c697d531cc5d270f339498b2c3c441b in 1292ms, sequenceid=325, compaction requested=false 2024-11-18T20:09:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:43,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-18T20:09:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-18T20:09:43,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-18T20:09:43,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3670 sec 2024-11-18T20:09:43,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.3740 sec 2024-11-18T20:09:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:44,945 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:44,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:09:44,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/fb7c5fd8fa974af1ab150ab11c464da5 is 50, key is test_row_0/A:col10/1731960584942/Put/seqid=0 2024-11-18T20:09:44,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742134_1310 (size=12301) 2024-11-18T20:09:44,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/fb7c5fd8fa974af1ab150ab11c464da5 2024-11-18T20:09:44,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/315262ec6d2f4e90be916898f1445e87 is 50, key is test_row_0/B:col10/1731960584942/Put/seqid=0 2024-11-18T20:09:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742135_1311 (size=12301) 2024-11-18T20:09:45,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/315262ec6d2f4e90be916898f1445e87 2024-11-18T20:09:45,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0967c97d8df64acb8969e764981ba655 is 50, key is test_row_0/C:col10/1731960584942/Put/seqid=0 2024-11-18T20:09:45,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742136_1312 (size=12301) 2024-11-18T20:09:45,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0967c97d8df64acb8969e764981ba655 2024-11-18T20:09:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/fb7c5fd8fa974af1ab150ab11c464da5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/fb7c5fd8fa974af1ab150ab11c464da5 2024-11-18T20:09:45,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960645134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:45,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/fb7c5fd8fa974af1ab150ab11c464da5, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T20:09:45,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/315262ec6d2f4e90be916898f1445e87 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/315262ec6d2f4e90be916898f1445e87 2024-11-18T20:09:45,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/315262ec6d2f4e90be916898f1445e87, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T20:09:45,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/0967c97d8df64acb8969e764981ba655 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0967c97d8df64acb8969e764981ba655 2024-11-18T20:09:45,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0967c97d8df64acb8969e764981ba655, entries=150, sequenceid=340, filesize=12.0 K 2024-11-18T20:09:45,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9c697d531cc5d270f339498b2c3c441b in 285ms, sequenceid=340, compaction requested=true 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:45,231 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:09:45,231 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:45,234 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:45,234 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:45,234 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:45,234 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9d2ae96514074af5a94a92dd77751519, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/12e486bc59cc49e09df5742483ab6079, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/fb7c5fd8fa974af1ab150ab11c464da5] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.8 K 2024-11-18T20:09:45,235 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2ae96514074af5a94a92dd77751519, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731960581513 2024-11-18T20:09:45,236 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e486bc59cc49e09df5742483ab6079, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731960581597 2024-11-18T20:09:45,236 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7c5fd8fa974af1ab150ab11c464da5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731960582775 2024-11-18T20:09:45,239 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:45,239 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:45,239 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:45,239 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/da3b0c9ab4c84d13b58dd096914b3185, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/42f46c7a71cd4896a47841dd99d2f1aa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/315262ec6d2f4e90be916898f1445e87] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.8 K 2024-11-18T20:09:45,241 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting da3b0c9ab4c84d13b58dd096914b3185, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731960581513 2024-11-18T20:09:45,241 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f46c7a71cd4896a47841dd99d2f1aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731960581597 2024-11-18T20:09:45,242 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 315262ec6d2f4e90be916898f1445e87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731960582775 2024-11-18T20:09:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:45,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:45,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:09:45,261 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:45,262 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/7fe0d72950e04273bbeaa46871a6af26 is 50, key is test_row_0/A:col10/1731960584942/Put/seqid=0 2024-11-18T20:09:45,269 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:45,270 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/82b606f81512482a9748bd8e5bfd289d is 50, key is test_row_0/B:col10/1731960584942/Put/seqid=0 2024-11-18T20:09:45,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/5c6aa6179fd54ab4908b7fdcc4302c5e is 50, key is test_row_0/A:col10/1731960585105/Put/seqid=0 2024-11-18T20:09:45,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742137_1313 (size=13153) 2024-11-18T20:09:45,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742138_1314 (size=13153) 2024-11-18T20:09:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742139_1315 (size=12301) 2024-11-18T20:09:45,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960645339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:45,350 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/82b606f81512482a9748bd8e5bfd289d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/82b606f81512482a9748bd8e5bfd289d 2024-11-18T20:09:45,357 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 82b606f81512482a9748bd8e5bfd289d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:45,357 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:45,357 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960585231; duration=0sec 2024-11-18T20:09:45,357 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:45,357 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:45,357 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:45,358 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:45,358 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:45,358 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:45,359 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/513a98ac0b954e20bba45f4158b6a144, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/121a51d7e981485ab11bb08efa244c33, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0967c97d8df64acb8969e764981ba655] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=36.8 K 2024-11-18T20:09:45,359 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 513a98ac0b954e20bba45f4158b6a144, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731960581513 2024-11-18T20:09:45,359 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 121a51d7e981485ab11bb08efa244c33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731960581597 2024-11-18T20:09:45,360 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0967c97d8df64acb8969e764981ba655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731960582775 2024-11-18T20:09:45,378 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:45,379 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c722f152ea7b4ba49504ae20f995956e is 50, key is test_row_0/C:col10/1731960584942/Put/seqid=0 2024-11-18T20:09:45,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742140_1316 (size=13153) 2024-11-18T20:09:45,400 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/c722f152ea7b4ba49504ae20f995956e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c722f152ea7b4ba49504ae20f995956e 2024-11-18T20:09:45,420 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into c722f152ea7b4ba49504ae20f995956e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:45,420 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:45,420 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960585231; duration=0sec 2024-11-18T20:09:45,421 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:45,421 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960645455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:45,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960645662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:45,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/5c6aa6179fd54ab4908b7fdcc4302c5e 2024-11-18T20:09:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:09:45,746 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:45,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:45,755 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/7fe0d72950e04273bbeaa46871a6af26 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7fe0d72950e04273bbeaa46871a6af26 2024-11-18T20:09:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-18T20:09:45,757 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:09:45,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:45,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:45,763 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 7fe0d72950e04273bbeaa46871a6af26(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:45,763 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:45,763 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960585231; duration=0sec 2024-11-18T20:09:45,763 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:45,763 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:45,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3e44b90a94664f1bb3157268041ce079 is 50, key is test_row_0/B:col10/1731960585105/Put/seqid=0 2024-11-18T20:09:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742141_1317 (size=12301) 2024-11-18T20:09:45,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3e44b90a94664f1bb3157268041ce079 2024-11-18T20:09:45,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b1fd1dfbbd054564b3ce9a7b84a11cdf is 50, key is test_row_0/C:col10/1731960585105/Put/seqid=0 2024-11-18T20:09:45,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742142_1318 (size=12301) 2024-11-18T20:09:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:09:45,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:09:45,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:09:45,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:45,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:45,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:45,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:45,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:45,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960645972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:09:46,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:09:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:46,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:09:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:46,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b1fd1dfbbd054564b3ce9a7b84a11cdf 2024-11-18T20:09:46,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/5c6aa6179fd54ab4908b7fdcc4302c5e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5c6aa6179fd54ab4908b7fdcc4302c5e 2024-11-18T20:09:46,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5c6aa6179fd54ab4908b7fdcc4302c5e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T20:09:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3e44b90a94664f1bb3157268041ce079 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3e44b90a94664f1bb3157268041ce079 2024-11-18T20:09:46,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3e44b90a94664f1bb3157268041ce079, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T20:09:46,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b1fd1dfbbd054564b3ce9a7b84a11cdf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b1fd1dfbbd054564b3ce9a7b84a11cdf 2024-11-18T20:09:46,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b1fd1dfbbd054564b3ce9a7b84a11cdf, entries=150, sequenceid=365, filesize=12.0 K 2024-11-18T20:09:46,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9c697d531cc5d270f339498b2c3c441b in 1015ms, sequenceid=365, compaction requested=false 2024-11-18T20:09:46,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:09:46,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:09:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,390 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:09:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/d35800a95d38416996a46df320df501f is 50, key is test_row_0/A:col10/1731960585336/Put/seqid=0 2024-11-18T20:09:46,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742143_1319 (size=9857) 2024-11-18T20:09:46,405 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/d35800a95d38416996a46df320df501f 2024-11-18T20:09:46,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/6b27294f34e8462fb1717230feacb47f is 50, key is test_row_0/B:col10/1731960585336/Put/seqid=0 2024-11-18T20:09:46,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742144_1320 (size=9857) 2024-11-18T20:09:46,446 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/6b27294f34e8462fb1717230feacb47f 2024-11-18T20:09:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/48edf8f6ce944eee8a6bb57ea36aa041 is 50, key is test_row_0/C:col10/1731960585336/Put/seqid=0 2024-11-18T20:09:46,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742145_1321 (size=9857) 2024-11-18T20:09:46,465 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/48edf8f6ce944eee8a6bb57ea36aa041 2024-11-18T20:09:46,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/d35800a95d38416996a46df320df501f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d35800a95d38416996a46df320df501f 2024-11-18T20:09:46,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d35800a95d38416996a46df320df501f, entries=100, sequenceid=379, filesize=9.6 K 2024-11-18T20:09:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/6b27294f34e8462fb1717230feacb47f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6b27294f34e8462fb1717230feacb47f 2024-11-18T20:09:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:46,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:46,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6b27294f34e8462fb1717230feacb47f, entries=100, sequenceid=379, filesize=9.6 K 2024-11-18T20:09:46,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/48edf8f6ce944eee8a6bb57ea36aa041 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/48edf8f6ce944eee8a6bb57ea36aa041 2024-11-18T20:09:46,500 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/48edf8f6ce944eee8a6bb57ea36aa041, entries=100, sequenceid=379, filesize=9.6 K 2024-11-18T20:09:46,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for 9c697d531cc5d270f339498b2c3c441b in 112ms, sequenceid=379, compaction requested=true 2024-11-18T20:09:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-18T20:09:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-18T20:09:46,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-18T20:09:46,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 744 msec 2024-11-18T20:09:46,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 755 msec 2024-11-18T20:09:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:46,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:09:46,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/8c3d4c342fd34f40a4035af0dde4d3ca is 50, key is test_row_0/A:col10/1731960586504/Put/seqid=0 2024-11-18T20:09:46,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742146_1322 (size=14741) 2024-11-18T20:09:46,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/8c3d4c342fd34f40a4035af0dde4d3ca 2024-11-18T20:09:46,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/1b564c155d5e404ba3a10719d592455b is 50, key is test_row_0/B:col10/1731960586504/Put/seqid=0 2024-11-18T20:09:46,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742147_1323 (size=12301) 2024-11-18T20:09:46,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/1b564c155d5e404ba3a10719d592455b 2024-11-18T20:09:46,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/3d86afbb7a77471b9f751c081bd7c912 is 50, key is test_row_0/C:col10/1731960586504/Put/seqid=0 2024-11-18T20:09:46,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742148_1324 (size=12301) 2024-11-18T20:09:46,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960646598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960646712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:09:46,892 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:46,893 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-18T20:09:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:09:46,895 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:46,897 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:46,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960646921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:46,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/3d86afbb7a77471b9f751c081bd7c912 2024-11-18T20:09:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/8c3d4c342fd34f40a4035af0dde4d3ca as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8c3d4c342fd34f40a4035af0dde4d3ca 2024-11-18T20:09:46,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8c3d4c342fd34f40a4035af0dde4d3ca, entries=200, sequenceid=390, filesize=14.4 K 2024-11-18T20:09:46,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/1b564c155d5e404ba3a10719d592455b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/1b564c155d5e404ba3a10719d592455b 2024-11-18T20:09:46,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/1b564c155d5e404ba3a10719d592455b, entries=150, sequenceid=390, filesize=12.0 K 2024-11-18T20:09:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/3d86afbb7a77471b9f751c081bd7c912 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3d86afbb7a77471b9f751c081bd7c912 2024-11-18T20:09:46,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3d86afbb7a77471b9f751c081bd7c912, entries=150, sequenceid=390, filesize=12.0 K 2024-11-18T20:09:46,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9c697d531cc5d270f339498b2c3c441b in 480ms, sequenceid=390, compaction requested=true 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:46,990 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:46,990 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:46,992 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:46,992 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:46,992 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:46,992 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:46,992 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,992 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:46,992 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7fe0d72950e04273bbeaa46871a6af26, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5c6aa6179fd54ab4908b7fdcc4302c5e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d35800a95d38416996a46df320df501f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8c3d4c342fd34f40a4035af0dde4d3ca] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=48.9 K 2024-11-18T20:09:46,992 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/82b606f81512482a9748bd8e5bfd289d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3e44b90a94664f1bb3157268041ce079, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6b27294f34e8462fb1717230feacb47f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/1b564c155d5e404ba3a10719d592455b] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=46.5 K 2024-11-18T20:09:46,992 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 82b606f81512482a9748bd8e5bfd289d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731960582775 2024-11-18T20:09:46,992 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe0d72950e04273bbeaa46871a6af26, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731960582775 2024-11-18T20:09:46,993 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e44b90a94664f1bb3157268041ce079, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731960585105 2024-11-18T20:09:46,993 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c6aa6179fd54ab4908b7fdcc4302c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731960585105 2024-11-18T20:09:46,993 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b27294f34e8462fb1717230feacb47f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731960585328 2024-11-18T20:09:46,993 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d35800a95d38416996a46df320df501f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731960585328 2024-11-18T20:09:46,993 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b564c155d5e404ba3a10719d592455b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731960586491 2024-11-18T20:09:46,993 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c3d4c342fd34f40a4035af0dde4d3ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731960586491 2024-11-18T20:09:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:09:47,006 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:47,006 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0a39e1ab46454a54a61ff85db5f96157 is 50, key is test_row_0/A:col10/1731960586504/Put/seqid=0 2024-11-18T20:09:47,019 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:47,019 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/00ed8d965e624af89653ef79e2e63c62 is 50, key is test_row_0/B:col10/1731960586504/Put/seqid=0 2024-11-18T20:09:47,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742149_1325 (size=13289) 2024-11-18T20:09:47,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T20:09:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:47,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:09:47,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742150_1326 (size=13289) 2024-11-18T20:09:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/e734ed5fa4da43f981dea9e7e2f9af67 is 50, key is test_row_0/A:col10/1731960586559/Put/seqid=0 2024-11-18T20:09:47,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742151_1327 (size=12301) 2024-11-18T20:09:47,127 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/e734ed5fa4da43f981dea9e7e2f9af67 2024-11-18T20:09:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/cbbd5e9718aa4db1bbff569dd6c1461e is 50, key is test_row_0/B:col10/1731960586559/Put/seqid=0 2024-11-18T20:09:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742152_1328 (size=12301) 2024-11-18T20:09:47,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/cbbd5e9718aa4db1bbff569dd6c1461e 2024-11-18T20:09:47,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/3275f767476c49c78e8b496c6e2e73f8 is 50, key is test_row_0/C:col10/1731960586559/Put/seqid=0 2024-11-18T20:09:47,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742153_1329 (size=12301) 2024-11-18T20:09:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:09:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:47,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960647276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:47,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960647392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:47,445 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/0a39e1ab46454a54a61ff85db5f96157 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0a39e1ab46454a54a61ff85db5f96157 2024-11-18T20:09:47,451 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 0a39e1ab46454a54a61ff85db5f96157(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:47,451 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:47,451 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=12, startTime=1731960586990; duration=0sec 2024-11-18T20:09:47,451 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:47,451 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:47,451 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:09:47,452 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:09:47,452 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:47,452 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:47,453 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c722f152ea7b4ba49504ae20f995956e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b1fd1dfbbd054564b3ce9a7b84a11cdf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/48edf8f6ce944eee8a6bb57ea36aa041, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3d86afbb7a77471b9f751c081bd7c912] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=46.5 K 2024-11-18T20:09:47,453 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c722f152ea7b4ba49504ae20f995956e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731960582775 2024-11-18T20:09:47,453 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fd1dfbbd054564b3ce9a7b84a11cdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731960585105 2024-11-18T20:09:47,454 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48edf8f6ce944eee8a6bb57ea36aa041, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731960585328 2024-11-18T20:09:47,454 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d86afbb7a77471b9f751c081bd7c912, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731960586491 2024-11-18T20:09:47,461 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:47,461 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/a9c6cbabef2a4631933ab99b93f2db9f is 50, key is test_row_0/C:col10/1731960586504/Put/seqid=0 2024-11-18T20:09:47,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742154_1330 (size=13289) 2024-11-18T20:09:47,479 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/00ed8d965e624af89653ef79e2e63c62 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/00ed8d965e624af89653ef79e2e63c62 2024-11-18T20:09:47,480 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/a9c6cbabef2a4631933ab99b93f2db9f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/a9c6cbabef2a4631933ab99b93f2db9f 2024-11-18T20:09:47,485 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 00ed8d965e624af89653ef79e2e63c62(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:47,486 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:47,486 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=12, startTime=1731960586990; duration=0sec 2024-11-18T20:09:47,486 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:47,486 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:47,486 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into a9c6cbabef2a4631933ab99b93f2db9f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:47,486 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:47,486 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=12, startTime=1731960586990; duration=0sec 2024-11-18T20:09:47,486 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:47,486 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:09:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960647602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:47,611 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/3275f767476c49c78e8b496c6e2e73f8 2024-11-18T20:09:47,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/e734ed5fa4da43f981dea9e7e2f9af67 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e734ed5fa4da43f981dea9e7e2f9af67 2024-11-18T20:09:47,620 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e734ed5fa4da43f981dea9e7e2f9af67, entries=150, sequenceid=415, filesize=12.0 K 2024-11-18T20:09:47,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/cbbd5e9718aa4db1bbff569dd6c1461e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/cbbd5e9718aa4db1bbff569dd6c1461e 2024-11-18T20:09:47,625 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/cbbd5e9718aa4db1bbff569dd6c1461e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-18T20:09:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/3275f767476c49c78e8b496c6e2e73f8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3275f767476c49c78e8b496c6e2e73f8 2024-11-18T20:09:47,629 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3275f767476c49c78e8b496c6e2e73f8, entries=150, sequenceid=415, filesize=12.0 K 2024-11-18T20:09:47,630 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9c697d531cc5d270f339498b2c3c441b in 571ms, sequenceid=415, compaction requested=false 2024-11-18T20:09:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-18T20:09:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-18T20:09:47,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-18T20:09:47,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 735 msec 2024-11-18T20:09:47,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 744 msec 2024-11-18T20:09:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:47,915 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:47,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:09:47,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/e94eb7de6e314e559fa8ef33ef81dc16 is 50, key is test_row_0/A:col10/1731960587261/Put/seqid=0 2024-11-18T20:09:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742155_1331 (size=12301) 2024-11-18T20:09:47,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/e94eb7de6e314e559fa8ef33ef81dc16 2024-11-18T20:09:47,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3b781dd6208942ab9c2273ec215cd71d is 50, key is test_row_0/B:col10/1731960587261/Put/seqid=0 2024-11-18T20:09:47,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742156_1332 (size=12301) 2024-11-18T20:09:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960648010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:09:48,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:48,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-18T20:09:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:09:48,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:48,037 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:48,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960648122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:09:48,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:09:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42112 deadline: 1731960648334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:09:48,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:09:48,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:09:48,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3b781dd6208942ab9c2273ec215cd71d 2024-11-18T20:09:48,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/ca955b9abd664632ae5659fbc98e55f3 is 50, key is test_row_0/C:col10/1731960587261/Put/seqid=0 2024-11-18T20:09:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742157_1333 (size=12301) 2024-11-18T20:09:48,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/ca955b9abd664632ae5659fbc98e55f3 2024-11-18T20:09:48,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/e94eb7de6e314e559fa8ef33ef81dc16 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e94eb7de6e314e559fa8ef33ef81dc16 2024-11-18T20:09:48,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e94eb7de6e314e559fa8ef33ef81dc16, entries=150, sequenceid=430, filesize=12.0 K 2024-11-18T20:09:48,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/3b781dd6208942ab9c2273ec215cd71d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3b781dd6208942ab9c2273ec215cd71d 2024-11-18T20:09:48,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3b781dd6208942ab9c2273ec215cd71d, entries=150, sequenceid=430, filesize=12.0 K 2024-11-18T20:09:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/ca955b9abd664632ae5659fbc98e55f3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/ca955b9abd664632ae5659fbc98e55f3 2024-11-18T20:09:48,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/ca955b9abd664632ae5659fbc98e55f3, entries=150, sequenceid=430, filesize=12.0 K 2024-11-18T20:09:48,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9c697d531cc5d270f339498b2c3c441b in 541ms, sequenceid=430, compaction requested=true 2024-11-18T20:09:48,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:48,457 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c697d531cc5d270f339498b2c3c441b:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:09:48,459 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:48,459 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/A is initiating minor compaction (all files) 2024-11-18T20:09:48,459 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/A in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,459 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0a39e1ab46454a54a61ff85db5f96157, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e734ed5fa4da43f981dea9e7e2f9af67, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e94eb7de6e314e559fa8ef33ef81dc16] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=37.0 K 2024-11-18T20:09:48,459 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:48,460 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a39e1ab46454a54a61ff85db5f96157, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731960586491 2024-11-18T20:09:48,460 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e734ed5fa4da43f981dea9e7e2f9af67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731960586559 2024-11-18T20:09:48,461 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e94eb7de6e314e559fa8ef33ef81dc16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731960587251 2024-11-18T20:09:48,469 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:48,470 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/8e5fadd9d1dc4e59ac4fe0731144ef26 is 50, key is test_row_0/A:col10/1731960587261/Put/seqid=0 2024-11-18T20:09:48,471 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:48,471 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/B is initiating minor compaction (all files) 2024-11-18T20:09:48,471 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/B in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,471 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/00ed8d965e624af89653ef79e2e63c62, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/cbbd5e9718aa4db1bbff569dd6c1461e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3b781dd6208942ab9c2273ec215cd71d] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=37.0 K 2024-11-18T20:09:48,475 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ed8d965e624af89653ef79e2e63c62, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731960586491 2024-11-18T20:09:48,475 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbbd5e9718aa4db1bbff569dd6c1461e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731960586559 2024-11-18T20:09:48,476 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b781dd6208942ab9c2273ec215cd71d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731960587251 2024-11-18T20:09:48,486 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:48,486 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/596ff9805f26429aaaa96550a607d5bc is 50, key is test_row_0/B:col10/1731960587261/Put/seqid=0 2024-11-18T20:09:48,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742158_1334 (size=13391) 2024-11-18T20:09:48,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:09:48,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:09:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/6af2b9695edf4e9d8536c8e7bf9e7660 is 50, key is test_row_0/A:col10/1731960588009/Put/seqid=0 2024-11-18T20:09:48,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742159_1335 (size=13391) 2024-11-18T20:09:48,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742160_1336 (size=12301) 2024-11-18T20:09:48,549 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/6af2b9695edf4e9d8536c8e7bf9e7660 2024-11-18T20:09:48,554 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-18T20:09:48,554 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-18T20:09:48,554 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:48,554 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:48,554 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,554 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,554 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,554 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:48,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:48,555 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-18T20:09:48,556 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:48,556 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,556 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:48,556 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-18T20:09:48,557 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:48,557 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,557 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-18T20:09:48,557 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:48,557 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:48,557 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:48,557 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/94c023752de941bd947a965588617268 is 50, key is test_row_0/B:col10/1731960588009/Put/seqid=0 2024-11-18T20:09:48,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742161_1337 (size=12301) 2024-11-18T20:09:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:48,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. as already flushing 2024-11-18T20:09:48,644 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-18T20:09:48,644 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:48,644 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,644 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:48,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:09:48,905 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/8e5fadd9d1dc4e59ac4fe0731144ef26 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8e5fadd9d1dc4e59ac4fe0731144ef26 2024-11-18T20:09:48,910 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/A of 9c697d531cc5d270f339498b2c3c441b into 8e5fadd9d1dc4e59ac4fe0731144ef26(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:48,910 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:48,910 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/A, priority=13, startTime=1731960588457; duration=0sec 2024-11-18T20:09:48,910 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:48,910 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:A 2024-11-18T20:09:48,910 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:48,911 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:48,911 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 9c697d531cc5d270f339498b2c3c441b/C is initiating minor compaction (all files) 2024-11-18T20:09:48,911 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c697d531cc5d270f339498b2c3c441b/C in TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:48,911 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/a9c6cbabef2a4631933ab99b93f2db9f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3275f767476c49c78e8b496c6e2e73f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/ca955b9abd664632ae5659fbc98e55f3] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp, totalSize=37.0 K 2024-11-18T20:09:48,911 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c6cbabef2a4631933ab99b93f2db9f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731960586491 2024-11-18T20:09:48,912 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3275f767476c49c78e8b496c6e2e73f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731960586559 2024-11-18T20:09:48,912 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ca955b9abd664632ae5659fbc98e55f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731960587251 2024-11-18T20:09:48,917 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c697d531cc5d270f339498b2c3c441b#C#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:48,917 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/673024f49dd24aeb8d54a0317109a7a1 is 50, key is test_row_0/C:col10/1731960587261/Put/seqid=0 2024-11-18T20:09:48,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742162_1338 (size=13391) 2024-11-18T20:09:48,938 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/596ff9805f26429aaaa96550a607d5bc as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/596ff9805f26429aaaa96550a607d5bc 2024-11-18T20:09:48,944 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/B of 9c697d531cc5d270f339498b2c3c441b into 596ff9805f26429aaaa96550a607d5bc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:48,944 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:48,945 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/B, priority=13, startTime=1731960588458; duration=0sec 2024-11-18T20:09:48,945 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:48,945 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:B 2024-11-18T20:09:48,971 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/94c023752de941bd947a965588617268 2024-11-18T20:09:48,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b08fe50c3ff4473da5241b75396378a3 is 50, key is test_row_0/C:col10/1731960588009/Put/seqid=0 2024-11-18T20:09:48,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742163_1339 (size=12301) 2024-11-18T20:09:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:09:49,328 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/673024f49dd24aeb8d54a0317109a7a1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/673024f49dd24aeb8d54a0317109a7a1 2024-11-18T20:09:49,335 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c697d531cc5d270f339498b2c3c441b/C of 9c697d531cc5d270f339498b2c3c441b into 673024f49dd24aeb8d54a0317109a7a1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:49,335 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:49,335 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., storeName=9c697d531cc5d270f339498b2c3c441b/C, priority=13, startTime=1731960588458; duration=0sec 2024-11-18T20:09:49,335 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:49,335 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c697d531cc5d270f339498b2c3c441b:C 2024-11-18T20:09:49,383 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b08fe50c3ff4473da5241b75396378a3 2024-11-18T20:09:49,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/6af2b9695edf4e9d8536c8e7bf9e7660 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6af2b9695edf4e9d8536c8e7bf9e7660 2024-11-18T20:09:49,392 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6af2b9695edf4e9d8536c8e7bf9e7660, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T20:09:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/94c023752de941bd947a965588617268 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/94c023752de941bd947a965588617268 2024-11-18T20:09:49,396 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/94c023752de941bd947a965588617268, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T20:09:49,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/b08fe50c3ff4473da5241b75396378a3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b08fe50c3ff4473da5241b75396378a3 2024-11-18T20:09:49,400 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b08fe50c3ff4473da5241b75396378a3, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T20:09:49,400 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 9c697d531cc5d270f339498b2c3c441b in 900ms, sequenceid=454, compaction requested=false 2024-11-18T20:09:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 9c697d531cc5d270f339498b2c3c441b: 2024-11-18T20:09:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-18T20:09:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-18T20:09:49,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-18T20:09:49,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3650 sec 2024-11-18T20:09:49,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.3700 sec 2024-11-18T20:09:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:09:50,182 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:50,233 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-18T20:09:50,233 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:50,233 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,233 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:50,254 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-18T20:09:50,254 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:50,254 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,254 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:50,295 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-18T20:09:50,295 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:50,295 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,295 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:50,306 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-18T20:09:50,306 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:09:50,306 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,306 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 256 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4041 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4096 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3942 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4054 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4094 2024-11-18T20:09:50,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:09:50,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:09:50,308 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:09:50,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:50,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:09:50,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:09:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:09:50,315 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960590315"}]},"ts":"1731960590315"} 2024-11-18T20:09:50,317 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:09:50,317 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:09:50,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:09:50,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, UNASSIGN}] 2024-11-18T20:09:50,321 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, UNASSIGN 2024-11-18T20:09:50,321 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=9c697d531cc5d270f339498b2c3c441b, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:50,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, UNASSIGN because future has completed 2024-11-18T20:09:50,323 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:09:50,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:09:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:09:50,476 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:50,476 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:09:50,477 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 9c697d531cc5d270f339498b2c3c441b, disabling compactions & flushes 2024-11-18T20:09:50,477 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:50,477 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:50,477 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. after waiting 0 ms 2024-11-18T20:09:50,477 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:50,477 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 9c697d531cc5d270f339498b2c3c441b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:09:50,484 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/61a20a14c70c4b349ae189644226bf1f is 50, key is test_row_0/A:col10/1731960590292/Put/seqid=0 2024-11-18T20:09:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742164_1340 (size=9857) 2024-11-18T20:09:50,489 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/61a20a14c70c4b349ae189644226bf1f 2024-11-18T20:09:50,497 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/ffa4f6f5b88c4b1cbc61dec150865aea is 50, key is test_row_0/B:col10/1731960590292/Put/seqid=0 2024-11-18T20:09:50,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742165_1341 (size=9857) 2024-11-18T20:09:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:09:50,901 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/ffa4f6f5b88c4b1cbc61dec150865aea 2024-11-18T20:09:50,908 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/02dbb5f883994ff4a3f9da22d6c47e4c is 50, key is test_row_0/C:col10/1731960590292/Put/seqid=0 2024-11-18T20:09:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742166_1342 (size=9857) 2024-11-18T20:09:50,913 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/02dbb5f883994ff4a3f9da22d6c47e4c 2024-11-18T20:09:50,918 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/A/61a20a14c70c4b349ae189644226bf1f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/61a20a14c70c4b349ae189644226bf1f 2024-11-18T20:09:50,922 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/61a20a14c70c4b349ae189644226bf1f, entries=100, sequenceid=465, filesize=9.6 K 2024-11-18T20:09:50,923 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/B/ffa4f6f5b88c4b1cbc61dec150865aea as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ffa4f6f5b88c4b1cbc61dec150865aea 2024-11-18T20:09:50,927 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ffa4f6f5b88c4b1cbc61dec150865aea, entries=100, sequenceid=465, filesize=9.6 K 2024-11-18T20:09:50,928 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/.tmp/C/02dbb5f883994ff4a3f9da22d6c47e4c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/02dbb5f883994ff4a3f9da22d6c47e4c 2024-11-18T20:09:50,933 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/02dbb5f883994ff4a3f9da22d6c47e4c, entries=100, sequenceid=465, filesize=9.6 K 2024-11-18T20:09:50,934 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9c697d531cc5d270f339498b2c3c441b in 456ms, sequenceid=465, compaction requested=true 2024-11-18T20:09:50,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/da05233e609741059005bce74deed6e3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5de3657e20464e6ea60042de1c606220, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2ae5366352b84777a22a738b8a83cf6d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/85bc19f60c60402aa918880b45f5a38b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9a868b4770fa4fdc9f2e3f905437aadd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7a7ba619e8584c49ac01c0607812c5f3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/15ddc76a3e50497cac1b79791a02e694, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0946542cb8e240af86ac7e3db713553e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d326e73684a14848a7e21100c6b0130a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0c71baae370f4ed7875bd590d5432d5b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6e1e8290144f4b6c99634ed74e3db33a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2679a2443a1743048c638eb7e6a969eb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0549f190ff114241b4e069e9a0dfb58e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/845a66e31b1a45b391be796add1e0361, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9174fbb108b14e9c8d94dbf9a8f0308e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/34c67d4220c247c2a659a90949af0da3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f691ff1c6394452bbfc10f7057ad5c51, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/1c978dcc9de74c4b9c93683f807424d1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/86c7a08b95094378b24cf0929d82f517, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f16decf02fc24a968b5d806e005663b5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/371d960060084c51851f23a933bfa674, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f56a38f49a4046dfa9053f04ea8dd8ca, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9d2ae96514074af5a94a92dd77751519, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/12e486bc59cc49e09df5742483ab6079, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7fe0d72950e04273bbeaa46871a6af26, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/fb7c5fd8fa974af1ab150ab11c464da5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5c6aa6179fd54ab4908b7fdcc4302c5e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d35800a95d38416996a46df320df501f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8c3d4c342fd34f40a4035af0dde4d3ca, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0a39e1ab46454a54a61ff85db5f96157, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e734ed5fa4da43f981dea9e7e2f9af67, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e94eb7de6e314e559fa8ef33ef81dc16] to archive 2024-11-18T20:09:50,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:09:50,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/da05233e609741059005bce74deed6e3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/da05233e609741059005bce74deed6e3 2024-11-18T20:09:50,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5de3657e20464e6ea60042de1c606220 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5de3657e20464e6ea60042de1c606220 2024-11-18T20:09:50,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2ae5366352b84777a22a738b8a83cf6d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2ae5366352b84777a22a738b8a83cf6d 2024-11-18T20:09:50,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/85bc19f60c60402aa918880b45f5a38b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/85bc19f60c60402aa918880b45f5a38b 2024-11-18T20:09:50,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9a868b4770fa4fdc9f2e3f905437aadd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9a868b4770fa4fdc9f2e3f905437aadd 2024-11-18T20:09:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:09:50,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7a7ba619e8584c49ac01c0607812c5f3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7a7ba619e8584c49ac01c0607812c5f3 2024-11-18T20:09:50,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/15ddc76a3e50497cac1b79791a02e694 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/15ddc76a3e50497cac1b79791a02e694 2024-11-18T20:09:50,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0946542cb8e240af86ac7e3db713553e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0946542cb8e240af86ac7e3db713553e 2024-11-18T20:09:50,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d326e73684a14848a7e21100c6b0130a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d326e73684a14848a7e21100c6b0130a 2024-11-18T20:09:50,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0c71baae370f4ed7875bd590d5432d5b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0c71baae370f4ed7875bd590d5432d5b 2024-11-18T20:09:50,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6e1e8290144f4b6c99634ed74e3db33a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6e1e8290144f4b6c99634ed74e3db33a 2024-11-18T20:09:50,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2679a2443a1743048c638eb7e6a969eb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/2679a2443a1743048c638eb7e6a969eb 2024-11-18T20:09:50,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0549f190ff114241b4e069e9a0dfb58e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0549f190ff114241b4e069e9a0dfb58e 2024-11-18T20:09:50,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/845a66e31b1a45b391be796add1e0361 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/845a66e31b1a45b391be796add1e0361 2024-11-18T20:09:50,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9174fbb108b14e9c8d94dbf9a8f0308e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9174fbb108b14e9c8d94dbf9a8f0308e 2024-11-18T20:09:50,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/34c67d4220c247c2a659a90949af0da3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/34c67d4220c247c2a659a90949af0da3 2024-11-18T20:09:50,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f691ff1c6394452bbfc10f7057ad5c51 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f691ff1c6394452bbfc10f7057ad5c51 2024-11-18T20:09:50,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/1c978dcc9de74c4b9c93683f807424d1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/1c978dcc9de74c4b9c93683f807424d1 2024-11-18T20:09:50,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/86c7a08b95094378b24cf0929d82f517 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/86c7a08b95094378b24cf0929d82f517 2024-11-18T20:09:50,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f16decf02fc24a968b5d806e005663b5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f16decf02fc24a968b5d806e005663b5 2024-11-18T20:09:50,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/371d960060084c51851f23a933bfa674 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/371d960060084c51851f23a933bfa674 2024-11-18T20:09:50,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f56a38f49a4046dfa9053f04ea8dd8ca to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/f56a38f49a4046dfa9053f04ea8dd8ca 2024-11-18T20:09:50,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9d2ae96514074af5a94a92dd77751519 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/9d2ae96514074af5a94a92dd77751519 2024-11-18T20:09:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/12e486bc59cc49e09df5742483ab6079 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/12e486bc59cc49e09df5742483ab6079 2024-11-18T20:09:50,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7fe0d72950e04273bbeaa46871a6af26 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/7fe0d72950e04273bbeaa46871a6af26 2024-11-18T20:09:50,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/fb7c5fd8fa974af1ab150ab11c464da5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/fb7c5fd8fa974af1ab150ab11c464da5 2024-11-18T20:09:50,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5c6aa6179fd54ab4908b7fdcc4302c5e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/5c6aa6179fd54ab4908b7fdcc4302c5e 2024-11-18T20:09:50,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d35800a95d38416996a46df320df501f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/d35800a95d38416996a46df320df501f 2024-11-18T20:09:50,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8c3d4c342fd34f40a4035af0dde4d3ca to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8c3d4c342fd34f40a4035af0dde4d3ca 2024-11-18T20:09:50,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0a39e1ab46454a54a61ff85db5f96157 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/0a39e1ab46454a54a61ff85db5f96157 2024-11-18T20:09:50,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e734ed5fa4da43f981dea9e7e2f9af67 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e734ed5fa4da43f981dea9e7e2f9af67 2024-11-18T20:09:50,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e94eb7de6e314e559fa8ef33ef81dc16 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/e94eb7de6e314e559fa8ef33ef81dc16 2024-11-18T20:09:50,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/5346356b671c43ab869fae4da71f947f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/e0991fdfea4949a78ac30430be067a2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4fd7cba07ada44679629ea488d9bab17, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4d0da3a63c5a4278a43977d52b942415, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eeefd4e4a9fd4b028845a3329a11afd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/023d525377d3403abd8e28d52feb6422, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/c0c190f70427423788639e8f875a9bfc, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/95a6fefc693448f5836b8c224a8c1592, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6928a4af540b433db34f573026c0948e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/679032947841480fbd51fc3a53b5539e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/dce3f5a8397b450ca80cd8b2a518e2bd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/abb0818965fb467da94e32e9c3108232, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/9fe4934ddfd04a8e957d2a694989e707, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97037468176743d1b945b640b36e920d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eba95a7ee251410da57749d7ab6e5013, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3a2d9a7925ee4250ac91ac1e46e64558, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97b126ce02d74297a54a7a65abd8480d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/19c63659074047278896b5550ff97ad5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/25805e4ccd3c441d94bcab3d269bfeb6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/755b2c36f490443f92fb809b81e8fcb8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/35f26f62d86f45a9aa6ed6d2074babaa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/da3b0c9ab4c84d13b58dd096914b3185, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ed05d8193736492cbefe1686e79e6bc0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/42f46c7a71cd4896a47841dd99d2f1aa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/82b606f81512482a9748bd8e5bfd289d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/315262ec6d2f4e90be916898f1445e87, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3e44b90a94664f1bb3157268041ce079, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6b27294f34e8462fb1717230feacb47f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/00ed8d965e624af89653ef79e2e63c62, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/1b564c155d5e404ba3a10719d592455b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/cbbd5e9718aa4db1bbff569dd6c1461e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3b781dd6208942ab9c2273ec215cd71d] to archive 2024-11-18T20:09:50,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:09:50,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/5346356b671c43ab869fae4da71f947f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/5346356b671c43ab869fae4da71f947f 2024-11-18T20:09:50,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/e0991fdfea4949a78ac30430be067a2c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/e0991fdfea4949a78ac30430be067a2c 2024-11-18T20:09:50,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4fd7cba07ada44679629ea488d9bab17 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4fd7cba07ada44679629ea488d9bab17 2024-11-18T20:09:50,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4d0da3a63c5a4278a43977d52b942415 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/4d0da3a63c5a4278a43977d52b942415 2024-11-18T20:09:50,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eeefd4e4a9fd4b028845a3329a11afd3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eeefd4e4a9fd4b028845a3329a11afd3 2024-11-18T20:09:50,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/023d525377d3403abd8e28d52feb6422 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/023d525377d3403abd8e28d52feb6422 2024-11-18T20:09:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/c0c190f70427423788639e8f875a9bfc to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/c0c190f70427423788639e8f875a9bfc 2024-11-18T20:09:50,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/95a6fefc693448f5836b8c224a8c1592 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/95a6fefc693448f5836b8c224a8c1592 2024-11-18T20:09:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6928a4af540b433db34f573026c0948e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6928a4af540b433db34f573026c0948e 2024-11-18T20:09:50,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/679032947841480fbd51fc3a53b5539e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/679032947841480fbd51fc3a53b5539e 2024-11-18T20:09:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/dce3f5a8397b450ca80cd8b2a518e2bd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/dce3f5a8397b450ca80cd8b2a518e2bd 2024-11-18T20:09:50,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/abb0818965fb467da94e32e9c3108232 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/abb0818965fb467da94e32e9c3108232 2024-11-18T20:09:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/9fe4934ddfd04a8e957d2a694989e707 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/9fe4934ddfd04a8e957d2a694989e707 2024-11-18T20:09:50,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97037468176743d1b945b640b36e920d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97037468176743d1b945b640b36e920d 2024-11-18T20:09:50,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eba95a7ee251410da57749d7ab6e5013 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/eba95a7ee251410da57749d7ab6e5013 2024-11-18T20:09:50,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3a2d9a7925ee4250ac91ac1e46e64558 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3a2d9a7925ee4250ac91ac1e46e64558 2024-11-18T20:09:50,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97b126ce02d74297a54a7a65abd8480d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/97b126ce02d74297a54a7a65abd8480d 2024-11-18T20:09:50,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/19c63659074047278896b5550ff97ad5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/19c63659074047278896b5550ff97ad5 2024-11-18T20:09:50,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/25805e4ccd3c441d94bcab3d269bfeb6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/25805e4ccd3c441d94bcab3d269bfeb6 2024-11-18T20:09:50,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/755b2c36f490443f92fb809b81e8fcb8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/755b2c36f490443f92fb809b81e8fcb8 2024-11-18T20:09:51,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/35f26f62d86f45a9aa6ed6d2074babaa to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/35f26f62d86f45a9aa6ed6d2074babaa 2024-11-18T20:09:51,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/da3b0c9ab4c84d13b58dd096914b3185 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/da3b0c9ab4c84d13b58dd096914b3185 2024-11-18T20:09:51,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ed05d8193736492cbefe1686e79e6bc0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ed05d8193736492cbefe1686e79e6bc0 2024-11-18T20:09:51,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/42f46c7a71cd4896a47841dd99d2f1aa to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/42f46c7a71cd4896a47841dd99d2f1aa 2024-11-18T20:09:51,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/82b606f81512482a9748bd8e5bfd289d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/82b606f81512482a9748bd8e5bfd289d 2024-11-18T20:09:51,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/315262ec6d2f4e90be916898f1445e87 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/315262ec6d2f4e90be916898f1445e87 2024-11-18T20:09:51,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3e44b90a94664f1bb3157268041ce079 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3e44b90a94664f1bb3157268041ce079 2024-11-18T20:09:51,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6b27294f34e8462fb1717230feacb47f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/6b27294f34e8462fb1717230feacb47f 2024-11-18T20:09:51,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/00ed8d965e624af89653ef79e2e63c62 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/00ed8d965e624af89653ef79e2e63c62 2024-11-18T20:09:51,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/1b564c155d5e404ba3a10719d592455b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/1b564c155d5e404ba3a10719d592455b 2024-11-18T20:09:51,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/cbbd5e9718aa4db1bbff569dd6c1461e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/cbbd5e9718aa4db1bbff569dd6c1461e 2024-11-18T20:09:51,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3b781dd6208942ab9c2273ec215cd71d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/3b781dd6208942ab9c2273ec215cd71d 2024-11-18T20:09:51,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9c5517f0873f416d83f971140a9053a6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/40258932ee2d4d0cb500c6b1ca1a2ae2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c6921e29c3bc4d2b95d63e2ccec55fdf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/556c9804f1d64555aa30b57c75984ef7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c790abf8e7bc4af2bec08b6b6caa7f96, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/d03fe842eba64ffa8b7d748791e55e09, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5095ade581264761b0224ac14fbbf09c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0d09895e4c794d97bab726b61c2c8fb0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/110a54661558488c87c483bc6621fe2c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/63e2378a76a84d54a018b94269525c10, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9a0c9260b68641bf814d62c340c7edf5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/171fece8dc934f16a1fc903cf2014e3f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/6ac9fea7799b416f92dfcdfc650081d7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/85052b62adb448fba0ca5af06688575d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/267ccf8cde7d472eb101cecd7d357836, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed717fb78ad4e088532dea956146dcf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/12303603965646b4b1c0ddf37b22fd0f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/8ca2d7a3fa594bdeab718374ad1684ba, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed46c0c42c3467ab0e37b4fabdc1d05, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/95959906818a4039a20d2db9a39b825f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/513a98ac0b954e20bba45f4158b6a144, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0ed7bd95660f42a8a7659077ca204a25, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/121a51d7e981485ab11bb08efa244c33, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c722f152ea7b4ba49504ae20f995956e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0967c97d8df64acb8969e764981ba655, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b1fd1dfbbd054564b3ce9a7b84a11cdf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/48edf8f6ce944eee8a6bb57ea36aa041, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/a9c6cbabef2a4631933ab99b93f2db9f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3d86afbb7a77471b9f751c081bd7c912, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3275f767476c49c78e8b496c6e2e73f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/ca955b9abd664632ae5659fbc98e55f3] to archive 2024-11-18T20:09:51,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:09:51,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9c5517f0873f416d83f971140a9053a6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9c5517f0873f416d83f971140a9053a6 2024-11-18T20:09:51,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/40258932ee2d4d0cb500c6b1ca1a2ae2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/40258932ee2d4d0cb500c6b1ca1a2ae2 2024-11-18T20:09:51,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c6921e29c3bc4d2b95d63e2ccec55fdf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c6921e29c3bc4d2b95d63e2ccec55fdf 2024-11-18T20:09:51,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/556c9804f1d64555aa30b57c75984ef7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/556c9804f1d64555aa30b57c75984ef7 2024-11-18T20:09:51,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c790abf8e7bc4af2bec08b6b6caa7f96 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c790abf8e7bc4af2bec08b6b6caa7f96 2024-11-18T20:09:51,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/d03fe842eba64ffa8b7d748791e55e09 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/d03fe842eba64ffa8b7d748791e55e09 2024-11-18T20:09:51,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5095ade581264761b0224ac14fbbf09c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5095ade581264761b0224ac14fbbf09c 2024-11-18T20:09:51,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0d09895e4c794d97bab726b61c2c8fb0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0d09895e4c794d97bab726b61c2c8fb0 2024-11-18T20:09:51,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/110a54661558488c87c483bc6621fe2c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/110a54661558488c87c483bc6621fe2c 2024-11-18T20:09:51,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/63e2378a76a84d54a018b94269525c10 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/63e2378a76a84d54a018b94269525c10 2024-11-18T20:09:51,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b7ab2bf8a9014dc2ab2c4b3833ee9f6c 2024-11-18T20:09:51,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9a0c9260b68641bf814d62c340c7edf5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/9a0c9260b68641bf814d62c340c7edf5 2024-11-18T20:09:51,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/171fece8dc934f16a1fc903cf2014e3f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/171fece8dc934f16a1fc903cf2014e3f 2024-11-18T20:09:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/6ac9fea7799b416f92dfcdfc650081d7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/6ac9fea7799b416f92dfcdfc650081d7 2024-11-18T20:09:51,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/85052b62adb448fba0ca5af06688575d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/85052b62adb448fba0ca5af06688575d 2024-11-18T20:09:51,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/267ccf8cde7d472eb101cecd7d357836 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/267ccf8cde7d472eb101cecd7d357836 2024-11-18T20:09:51,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed717fb78ad4e088532dea956146dcf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed717fb78ad4e088532dea956146dcf 2024-11-18T20:09:51,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/12303603965646b4b1c0ddf37b22fd0f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/12303603965646b4b1c0ddf37b22fd0f 2024-11-18T20:09:51,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/8ca2d7a3fa594bdeab718374ad1684ba to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/8ca2d7a3fa594bdeab718374ad1684ba 2024-11-18T20:09:51,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed46c0c42c3467ab0e37b4fabdc1d05 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/5ed46c0c42c3467ab0e37b4fabdc1d05 2024-11-18T20:09:51,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/95959906818a4039a20d2db9a39b825f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/95959906818a4039a20d2db9a39b825f 2024-11-18T20:09:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/513a98ac0b954e20bba45f4158b6a144 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/513a98ac0b954e20bba45f4158b6a144 2024-11-18T20:09:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0ed7bd95660f42a8a7659077ca204a25 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0ed7bd95660f42a8a7659077ca204a25 2024-11-18T20:09:51,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/121a51d7e981485ab11bb08efa244c33 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/121a51d7e981485ab11bb08efa244c33 2024-11-18T20:09:51,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c722f152ea7b4ba49504ae20f995956e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/c722f152ea7b4ba49504ae20f995956e 2024-11-18T20:09:51,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0967c97d8df64acb8969e764981ba655 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/0967c97d8df64acb8969e764981ba655 2024-11-18T20:09:51,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b1fd1dfbbd054564b3ce9a7b84a11cdf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b1fd1dfbbd054564b3ce9a7b84a11cdf 2024-11-18T20:09:51,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/48edf8f6ce944eee8a6bb57ea36aa041 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/48edf8f6ce944eee8a6bb57ea36aa041 2024-11-18T20:09:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/a9c6cbabef2a4631933ab99b93f2db9f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/a9c6cbabef2a4631933ab99b93f2db9f 2024-11-18T20:09:51,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3d86afbb7a77471b9f751c081bd7c912 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3d86afbb7a77471b9f751c081bd7c912 2024-11-18T20:09:51,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3275f767476c49c78e8b496c6e2e73f8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/3275f767476c49c78e8b496c6e2e73f8 2024-11-18T20:09:51,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/ca955b9abd664632ae5659fbc98e55f3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/ca955b9abd664632ae5659fbc98e55f3 2024-11-18T20:09:51,051 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=1 2024-11-18T20:09:51,052 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. 2024-11-18T20:09:51,052 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 9c697d531cc5d270f339498b2c3c441b: Waiting for close lock at 1731960590476Running coprocessor pre-close hooks at 1731960590476Disabling compacts and flushes for region at 1731960590476Disabling writes for close at 1731960590477 (+1 ms)Obtaining lock to block concurrent updates at 1731960590477Preparing flush snapshotting stores in 9c697d531cc5d270f339498b2c3c441b at 1731960590477Finished memstore snapshotting TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960590478 (+1 ms)Flushing stores of TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b. at 1731960590479 (+1 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/A: creating writer at 1731960590479Flushing 9c697d531cc5d270f339498b2c3c441b/A: appending metadata at 1731960590483 (+4 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/A: closing flushed file at 1731960590483Flushing 9c697d531cc5d270f339498b2c3c441b/B: creating writer at 1731960590493 (+10 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/B: appending metadata at 1731960590496 (+3 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/B: closing flushed file at 1731960590497 (+1 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/C: creating writer at 1731960590906 (+409 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/C: appending metadata at 1731960590908 (+2 ms)Flushing 9c697d531cc5d270f339498b2c3c441b/C: closing flushed file at 1731960590908Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e6f7d60: reopening flushed file at 1731960590917 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16a88a1a: reopening flushed file at 1731960590922 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18acf17e: reopening flushed file at 1731960590928 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9c697d531cc5d270f339498b2c3c441b in 456ms, sequenceid=465, compaction requested=true at 1731960590934 (+6 ms)Writing region close event to WAL at 1731960591048 (+114 ms)Running coprocessor post-close hooks at 1731960591052 (+4 ms)Closed at 1731960591052 2024-11-18T20:09:51,054 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:51,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=9c697d531cc5d270f339498b2c3c441b, regionState=CLOSED 2024-11-18T20:09:51,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:09:51,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-18T20:09:51,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c697d531cc5d270f339498b2c3c441b, server=d8f95bb18b58,42447,1731960497538 in 735 msec 2024-11-18T20:09:51,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-18T20:09:51,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c697d531cc5d270f339498b2c3c441b, UNASSIGN in 740 msec 2024-11-18T20:09:51,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-18T20:09:51,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 745 msec 2024-11-18T20:09:51,068 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960591067"}]},"ts":"1731960591067"} 2024-11-18T20:09:51,069 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:09:51,069 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:09:51,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 761 msec 2024-11-18T20:09:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:09:51,452 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:51,453 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:09:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:09:51,455 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,457 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:51,459 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/recovered.edits] 2024-11-18T20:09:51,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/61a20a14c70c4b349ae189644226bf1f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/61a20a14c70c4b349ae189644226bf1f 2024-11-18T20:09:51,462 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6af2b9695edf4e9d8536c8e7bf9e7660 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/6af2b9695edf4e9d8536c8e7bf9e7660 2024-11-18T20:09:51,463 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8e5fadd9d1dc4e59ac4fe0731144ef26 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/A/8e5fadd9d1dc4e59ac4fe0731144ef26 2024-11-18T20:09:51,465 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/596ff9805f26429aaaa96550a607d5bc to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/596ff9805f26429aaaa96550a607d5bc 2024-11-18T20:09:51,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/94c023752de941bd947a965588617268 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/94c023752de941bd947a965588617268 2024-11-18T20:09:51,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ffa4f6f5b88c4b1cbc61dec150865aea to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/B/ffa4f6f5b88c4b1cbc61dec150865aea 2024-11-18T20:09:51,468 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/02dbb5f883994ff4a3f9da22d6c47e4c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/02dbb5f883994ff4a3f9da22d6c47e4c 2024-11-18T20:09:51,469 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/673024f49dd24aeb8d54a0317109a7a1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/673024f49dd24aeb8d54a0317109a7a1 2024-11-18T20:09:51,470 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b08fe50c3ff4473da5241b75396378a3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/C/b08fe50c3ff4473da5241b75396378a3 2024-11-18T20:09:51,472 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/recovered.edits/468.seqid to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b/recovered.edits/468.seqid 2024-11-18T20:09:51,473 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/9c697d531cc5d270f339498b2c3c441b 2024-11-18T20:09:51,473 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:09:51,475 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,478 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:09:51,480 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:09:51,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:09:51,481 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960591481"}]},"ts":"9223372036854775807"} 2024-11-18T20:09:51,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:09:51,484 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9c697d531cc5d270f339498b2c3c441b, NAME => 'TestAcidGuarantees,,1731960566330.9c697d531cc5d270f339498b2c3c441b.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:09:51,484 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:09:51,484 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960591484"}]},"ts":"9223372036854775807"} 2024-11-18T20:09:51,486 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:09:51,487 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-18T20:09:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:09:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:09:51,565 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:51,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1249326010_22 at /127.0.0.1:58132 [Waiting for operation #361] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1190570139_22 at /127.0.0.1:58290 [Waiting for operation #292] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=484 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=1964 (was 1163) - AvailableMemoryMB LEAK? - 2024-11-18T20:09:51,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=484, ProcessCount=11, AvailableMemoryMB=1962 2024-11-18T20:09:51,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:09:51,582 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:09:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:51,584 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:09:51,584 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:51,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-18T20:09:51,585 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:09:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T20:09:51,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742167_1343 (size=959) 2024-11-18T20:09:51,592 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:09:51,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742168_1344 (size=53) 2024-11-18T20:09:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T20:09:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T20:09:52,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:09:52,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c01d05a23f041191706ac16b56e11567, disabling compactions & flushes 2024-11-18T20:09:52,002 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. after waiting 0 ms 2024-11-18T20:09:52,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,002 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c01d05a23f041191706ac16b56e11567: Waiting for close lock at 1731960592002Disabling compacts and flushes for region at 1731960592002Disabling writes for close at 1731960592002Writing region close event to WAL at 1731960592002Closed at 1731960592002 2024-11-18T20:09:52,003 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:09:52,003 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960592003"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960592003"}]},"ts":"1731960592003"} 2024-11-18T20:09:52,005 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:09:52,006 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:09:52,006 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960592006"}]},"ts":"1731960592006"} 2024-11-18T20:09:52,007 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:09:52,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, ASSIGN}] 2024-11-18T20:09:52,008 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, ASSIGN 2024-11-18T20:09:52,009 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, ASSIGN; state=OFFLINE, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=false 2024-11-18T20:09:52,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:52,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, ASSIGN because future has completed 2024-11-18T20:09:52,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:09:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T20:09:52,316 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,316 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:09:52,316 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,317 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:09:52,317 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,317 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,318 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,319 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01d05a23f041191706ac16b56e11567 columnFamilyName A 2024-11-18T20:09:52,319 DEBUG [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:52,320 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(327): Store=c01d05a23f041191706ac16b56e11567/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:52,320 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,321 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01d05a23f041191706ac16b56e11567 columnFamilyName B 2024-11-18T20:09:52,321 DEBUG [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:52,321 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(327): Store=c01d05a23f041191706ac16b56e11567/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:52,321 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,322 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01d05a23f041191706ac16b56e11567 columnFamilyName C 2024-11-18T20:09:52,322 DEBUG [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:52,323 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(327): Store=c01d05a23f041191706ac16b56e11567/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:52,323 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,324 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,324 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,325 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,325 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,325 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:09:52,326 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,327 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:09:52,328 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened c01d05a23f041191706ac16b56e11567; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72855348, jitterRate=0.08562928438186646}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:09:52,328 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:52,328 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for c01d05a23f041191706ac16b56e11567: Running coprocessor pre-open hook at 1731960592317Writing region info on filesystem at 1731960592317Initializing all the Stores at 1731960592317Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960592317Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960592318 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960592318Cleaning up temporary data from old regions at 1731960592325 (+7 ms)Running coprocessor post-open hooks at 1731960592328 (+3 ms)Region opened successfully at 1731960592328 2024-11-18T20:09:52,329 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., pid=89, masterSystemTime=1731960592313 2024-11-18T20:09:52,330 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,330 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:52,331 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:52,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:09:52,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-18T20:09:52,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 in 171 msec 2024-11-18T20:09:52,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-18T20:09:52,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, ASSIGN in 327 msec 2024-11-18T20:09:52,336 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:09:52,336 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960592336"}]},"ts":"1731960592336"} 2024-11-18T20:09:52,338 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:09:52,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:09:52,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 757 msec 2024-11-18T20:09:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T20:09:52,722 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:52,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:52,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:52,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:52,724 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5cf73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:52,726 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc566cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:52,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:52,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:52,729 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:09:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:09:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@711adde0 2024-11-18T20:09:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:09:52,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:09:52,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:09:52,732 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:09:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:09:52,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742169_1345 (size=995) 2024-11-18T20:09:53,145 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-18T20:09:53,145 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-18T20:09:53,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:09:53,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, REOPEN/MOVE}] 2024-11-18T20:09:53,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, REOPEN/MOVE 2024-11-18T20:09:53,153 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, REOPEN/MOVE because future has completed 2024-11-18T20:09:53,155 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:09:53,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:09:53,308 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,308 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:09:53,308 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing c01d05a23f041191706ac16b56e11567, disabling compactions & flushes 2024-11-18T20:09:53,308 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,308 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,308 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. after waiting 0 ms 2024-11-18T20:09:53,308 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,312 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:09:53,313 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,313 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for c01d05a23f041191706ac16b56e11567: Waiting for close lock at 1731960593308Running coprocessor pre-close hooks at 1731960593308Disabling compacts and flushes for region at 1731960593308Disabling writes for close at 1731960593308Writing region close event to WAL at 1731960593308Running coprocessor post-close hooks at 1731960593313 (+5 ms)Closed at 1731960593313 2024-11-18T20:09:53,313 WARN [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: c01d05a23f041191706ac16b56e11567 to self. 2024-11-18T20:09:53,315 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,315 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=CLOSED 2024-11-18T20:09:53,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:09:53,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-18T20:09:53,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 in 162 msec 2024-11-18T20:09:53,320 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, REOPEN/MOVE; state=CLOSED, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=true 2024-11-18T20:09:53,471 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, REOPEN/MOVE because future has completed 2024-11-18T20:09:53,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:09:53,627 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,627 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:09:53,627 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,627 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:09:53,627 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,627 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,629 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,630 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01d05a23f041191706ac16b56e11567 columnFamilyName A 2024-11-18T20:09:53,631 DEBUG [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:53,631 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(327): Store=c01d05a23f041191706ac16b56e11567/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:53,632 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,632 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01d05a23f041191706ac16b56e11567 columnFamilyName B 2024-11-18T20:09:53,632 DEBUG [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:53,633 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(327): Store=c01d05a23f041191706ac16b56e11567/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:53,633 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,634 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01d05a23f041191706ac16b56e11567 columnFamilyName C 2024-11-18T20:09:53,634 DEBUG [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:53,634 INFO [StoreOpener-c01d05a23f041191706ac16b56e11567-1 {}] regionserver.HStore(327): Store=c01d05a23f041191706ac16b56e11567/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:09:53,634 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,635 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,636 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,637 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,637 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,638 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:09:53,639 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,640 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened c01d05a23f041191706ac16b56e11567; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67200044, jitterRate=0.0013586878776550293}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:09:53,640 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,641 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for c01d05a23f041191706ac16b56e11567: Running coprocessor pre-open hook at 1731960593628Writing region info on filesystem at 1731960593628Initializing all the Stores at 1731960593628Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960593628Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960593629 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960593629Cleaning up temporary data from old regions at 1731960593637 (+8 ms)Running coprocessor post-open hooks at 1731960593640 (+3 ms)Region opened successfully at 1731960593640 2024-11-18T20:09:53,641 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., pid=94, masterSystemTime=1731960593624 2024-11-18T20:09:53,643 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,643 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,643 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=OPEN, openSeqNum=5, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:09:53,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-18T20:09:53,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 in 173 msec 2024-11-18T20:09:53,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-18T20:09:53,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, REOPEN/MOVE in 497 msec 2024-11-18T20:09:53,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-18T20:09:53,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-18T20:09:53,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-18T20:09:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-18T20:09:53,655 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:53,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,657 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fa4532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,659 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dcef32d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,661 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e140785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3aeaec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3727f402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21305aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,669 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea0d28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d61d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284127c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbb8f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,678 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72260dae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f940fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,681 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e08d069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,683 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aca0fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,685 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69adefdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7344faa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aebb657, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ce74ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:09:53,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:09:53,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:09:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:09:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:09:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e2560f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:09:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:09:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:09:53,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:09:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b31e837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:09:53,696 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,696 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,697 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,698 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,698 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,700 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,707 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,718 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,719 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-18T20:09:53,723 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:53,724 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,725 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:53,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:53,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T20:09:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:53,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:09:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,745 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:09:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:09:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:09:53,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:09:53,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:09:53,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f0a3963bd2f04b50b73edcafd88ff826_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960593722/Put/seqid=0 2024-11-18T20:09:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960653800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960653800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742170_1346 (size=12154) 2024-11-18T20:09:53,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960653803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,817 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960653804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960653806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T20:09:53,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f0a3963bd2f04b50b73edcafd88ff826_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f0a3963bd2f04b50b73edcafd88ff826_c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:53,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/933b93a49e0a46639c76954ce7cc5a95, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:53,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/933b93a49e0a46639c76954ce7cc5a95 is 175, key is test_row_0/A:col10/1731960593722/Put/seqid=0 2024-11-18T20:09:53,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742171_1347 (size=30955) 2024-11-18T20:09:53,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/933b93a49e0a46639c76954ce7cc5a95 2024-11-18T20:09:53,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T20:09:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:53,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:53,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a0cb1a66f0744effbbd76d303ef397e6 is 50, key is test_row_0/B:col10/1731960593722/Put/seqid=0 2024-11-18T20:09:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960653918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960653922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960653922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960653922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742172_1348 (size=12001) 2024-11-18T20:09:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a0cb1a66f0744effbbd76d303ef397e6 2024-11-18T20:09:53,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960653936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:53,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/bc610d1e356e4bd29bf0d77cf4349f21 is 50, key is test_row_0/C:col10/1731960593722/Put/seqid=0 2024-11-18T20:09:54,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T20:09:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742173_1349 (size=12001) 2024-11-18T20:09:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T20:09:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960654131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960654142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960654142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960654152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960654152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T20:09:54,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:54,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T20:09:54,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T20:09:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:54,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/bc610d1e356e4bd29bf0d77cf4349f21 2024-11-18T20:09:54,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/933b93a49e0a46639c76954ce7cc5a95 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95 2024-11-18T20:09:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960654442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95, entries=150, sequenceid=15, filesize=30.2 K 2024-11-18T20:09:54,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a0cb1a66f0744effbbd76d303ef397e6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a0cb1a66f0744effbbd76d303ef397e6 2024-11-18T20:09:54,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960654452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960654452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a0cb1a66f0744effbbd76d303ef397e6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:09:54,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/bc610d1e356e4bd29bf0d77cf4349f21 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/bc610d1e356e4bd29bf0d77cf4349f21 2024-11-18T20:09:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960654462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960654463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/bc610d1e356e4bd29bf0d77cf4349f21, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:09:54,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c01d05a23f041191706ac16b56e11567 in 754ms, sequenceid=15, compaction requested=false 2024-11-18T20:09:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:54,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T20:09:54,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:54,514 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:09:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187600ff26e21541379b9b7ba57b46af46_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960593802/Put/seqid=0 2024-11-18T20:09:54,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742174_1350 (size=12154) 2024-11-18T20:09:54,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:54,569 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187600ff26e21541379b9b7ba57b46af46_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187600ff26e21541379b9b7ba57b46af46_c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5a43d9f9c77743938059459dfacf81a2, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:54,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5a43d9f9c77743938059459dfacf81a2 is 175, key is test_row_0/A:col10/1731960593802/Put/seqid=0 2024-11-18T20:09:54,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742175_1351 (size=30955) 2024-11-18T20:09:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T20:09:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960654971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960654972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,978 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5a43d9f9c77743938059459dfacf81a2 2024-11-18T20:09:54,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960654975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/6dd1d55867584f6fb84be829a5855286 is 50, key is test_row_0/B:col10/1731960593802/Put/seqid=0 2024-11-18T20:09:54,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960654982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960654982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742176_1352 (size=12001) 2024-11-18T20:09:55,006 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/6dd1d55867584f6fb84be829a5855286 2024-11-18T20:09:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/8289c9ba10c44e0597fa3527ef5db54c is 50, key is test_row_0/C:col10/1731960593802/Put/seqid=0 2024-11-18T20:09:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742177_1353 (size=12001) 2024-11-18T20:09:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960655082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960655082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960655292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960655292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,423 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/8289c9ba10c44e0597fa3527ef5db54c 2024-11-18T20:09:55,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5a43d9f9c77743938059459dfacf81a2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2 2024-11-18T20:09:55,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2, entries=150, sequenceid=40, filesize=30.2 K 2024-11-18T20:09:55,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/6dd1d55867584f6fb84be829a5855286 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/6dd1d55867584f6fb84be829a5855286 2024-11-18T20:09:55,442 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/6dd1d55867584f6fb84be829a5855286, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:09:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/8289c9ba10c44e0597fa3527ef5db54c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8289c9ba10c44e0597fa3527ef5db54c 2024-11-18T20:09:55,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8289c9ba10c44e0597fa3527ef5db54c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:09:55,481 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c01d05a23f041191706ac16b56e11567 in 967ms, sequenceid=40, compaction requested=false 2024-11-18T20:09:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-18T20:09:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-18T20:09:55,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-18T20:09:55,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7580 sec 2024-11-18T20:09:55,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.7660 sec 2024-11-18T20:09:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:55,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:09:55,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180f8eb6b39ed8479eb292f023b28b93bd_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:55,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742178_1354 (size=17034) 2024-11-18T20:09:55,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:55,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180f8eb6b39ed8479eb292f023b28b93bd_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180f8eb6b39ed8479eb292f023b28b93bd_c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:55,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/718d15048ffe47919bf59e0fc55f1833, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:55,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/718d15048ffe47919bf59e0fc55f1833 is 175, key is test_row_0/A:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:55,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742179_1355 (size=48139) 2024-11-18T20:09:55,676 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/718d15048ffe47919bf59e0fc55f1833 2024-11-18T20:09:55,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/0c60acaf0c8f4aa8885449888f825be6 is 50, key is test_row_0/B:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742180_1356 (size=12001) 2024-11-18T20:09:55,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960655733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960655733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960655854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960655855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T20:09:55,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:55,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-18T20:09:55,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:09:55,886 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:55,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:09:55,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960655992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,996 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:55,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960655992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:55,998 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960656004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,009 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:09:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960656075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960656076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/0c60acaf0c8f4aa8885449888f825be6 2024-11-18T20:09:56,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/5061524600a14981ace2c59cc5a71e71 is 50, key is test_row_0/C:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:56,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742181_1357 (size=12001) 2024-11-18T20:09:56,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:09:56,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:56,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:09:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:09:56,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:56,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960656392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960656396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:09:56,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:09:56,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:56,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:56,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:09:56,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/5061524600a14981ace2c59cc5a71e71 2024-11-18T20:09:56,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/718d15048ffe47919bf59e0fc55f1833 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833 2024-11-18T20:09:56,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833, entries=250, sequenceid=53, filesize=47.0 K 2024-11-18T20:09:56,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/0c60acaf0c8f4aa8885449888f825be6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0c60acaf0c8f4aa8885449888f825be6 2024-11-18T20:09:56,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0c60acaf0c8f4aa8885449888f825be6, entries=150, sequenceid=53, filesize=11.7 K 2024-11-18T20:09:56,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/5061524600a14981ace2c59cc5a71e71 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/5061524600a14981ace2c59cc5a71e71 2024-11-18T20:09:56,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/5061524600a14981ace2c59cc5a71e71, entries=150, sequenceid=53, filesize=11.7 K 2024-11-18T20:09:56,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for c01d05a23f041191706ac16b56e11567 in 1075ms, sequenceid=53, compaction requested=true 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:56,685 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:56,685 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:56,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T20:09:56,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,689 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:09:56,698 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:56,698 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:56,698 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:09:56,698 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:09:56,699 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,699 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,699 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=107.5 K 2024-11-18T20:09:56,699 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a0cb1a66f0744effbbd76d303ef397e6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/6dd1d55867584f6fb84be829a5855286, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0c60acaf0c8f4aa8885449888f825be6] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=35.2 K 2024-11-18T20:09:56,699 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,699 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833] 2024-11-18T20:09:56,700 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a0cb1a66f0744effbbd76d303ef397e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960593711 2024-11-18T20:09:56,701 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933b93a49e0a46639c76954ce7cc5a95, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960593711 2024-11-18T20:09:56,703 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dd1d55867584f6fb84be829a5855286, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960593799 2024-11-18T20:09:56,706 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c60acaf0c8f4aa8885449888f825be6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960594974 2024-11-18T20:09:56,708 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a43d9f9c77743938059459dfacf81a2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960593799 2024-11-18T20:09:56,711 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 718d15048ffe47919bf59e0fc55f1833, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960594968 2024-11-18T20:09:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111831b52f37e84a4b918573267b08f987db_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960595716/Put/seqid=0 2024-11-18T20:09:56,725 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:56,726 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a85f6a3aa36546a2804eadfc9414ddef is 50, key is test_row_0/B:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:56,745 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:56,776 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187018278260e44f1c8c1d2211498c7080_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:56,778 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187018278260e44f1c8c1d2211498c7080_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:56,778 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187018278260e44f1c8c1d2211498c7080_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742183_1359 (size=12104) 2024-11-18T20:09:56,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742182_1358 (size=12154) 2024-11-18T20:09:56,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a85f6a3aa36546a2804eadfc9414ddef as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a85f6a3aa36546a2804eadfc9414ddef 2024-11-18T20:09:56,856 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into a85f6a3aa36546a2804eadfc9414ddef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:56,856 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:56,856 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=13, startTime=1731960596685; duration=0sec 2024-11-18T20:09:56,856 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:56,857 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:09:56,857 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:56,858 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:56,858 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:09:56,858 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:56,858 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/bc610d1e356e4bd29bf0d77cf4349f21, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8289c9ba10c44e0597fa3527ef5db54c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/5061524600a14981ace2c59cc5a71e71] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=35.2 K 2024-11-18T20:09:56,859 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting bc610d1e356e4bd29bf0d77cf4349f21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960593711 2024-11-18T20:09:56,859 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8289c9ba10c44e0597fa3527ef5db54c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960593799 2024-11-18T20:09:56,859 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5061524600a14981ace2c59cc5a71e71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960594974 2024-11-18T20:09:56,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742184_1360 (size=4469) 2024-11-18T20:09:56,885 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:56,886 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/1c71af87055f44f9a559317d2907e9e3 is 50, key is test_row_0/C:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:56,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742185_1361 (size=12104) 2024-11-18T20:09:56,971 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/1c71af87055f44f9a559317d2907e9e3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1c71af87055f44f9a559317d2907e9e3 2024-11-18T20:09:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960656972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:56,990 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into 1c71af87055f44f9a559317d2907e9e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:56,990 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:56,990 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=13, startTime=1731960596685; duration=0sec 2024-11-18T20:09:56,990 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:56,990 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:09:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960656979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:09:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960657092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960657102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:57,229 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111831b52f37e84a4b918573267b08f987db_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111831b52f37e84a4b918573267b08f987db_c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:57,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/1243b4cb74a643cbababa1f2cda17b09, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/1243b4cb74a643cbababa1f2cda17b09 is 175, key is test_row_0/A:col10/1731960595716/Put/seqid=0 2024-11-18T20:09:57,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742186_1362 (size=30955) 2024-11-18T20:09:57,279 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#311 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:57,280 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ec5a86d8bbdc47359fc41c2cd90c37b5 is 175, key is test_row_0/A:col10/1731960595607/Put/seqid=0 2024-11-18T20:09:57,284 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/1243b4cb74a643cbababa1f2cda17b09 2024-11-18T20:09:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742187_1363 (size=31058) 2024-11-18T20:09:57,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/7540c2a7210e4024a9448dbcd0d23710 is 50, key is test_row_0/B:col10/1731960595716/Put/seqid=0 2024-11-18T20:09:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960657303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,314 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ec5a86d8bbdc47359fc41c2cd90c37b5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ec5a86d8bbdc47359fc41c2cd90c37b5 2024-11-18T20:09:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960657322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,341 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into ec5a86d8bbdc47359fc41c2cd90c37b5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:57,341 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:57,341 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=13, startTime=1731960596685; duration=0sec 2024-11-18T20:09:57,341 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:57,341 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:09:57,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742188_1364 (size=12001) 2024-11-18T20:09:57,373 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/7540c2a7210e4024a9448dbcd0d23710 2024-11-18T20:09:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/4d626aede5dd49179e6678f5cb4b5444 is 50, key is test_row_0/C:col10/1731960595716/Put/seqid=0 2024-11-18T20:09:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742189_1365 (size=12001) 2024-11-18T20:09:57,499 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/4d626aede5dd49179e6678f5cb4b5444 2024-11-18T20:09:57,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/1243b4cb74a643cbababa1f2cda17b09 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09 2024-11-18T20:09:57,575 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09, entries=150, sequenceid=77, filesize=30.2 K 2024-11-18T20:09:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/7540c2a7210e4024a9448dbcd0d23710 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/7540c2a7210e4024a9448dbcd0d23710 2024-11-18T20:09:57,593 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/7540c2a7210e4024a9448dbcd0d23710, entries=150, sequenceid=77, filesize=11.7 K 2024-11-18T20:09:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/4d626aede5dd49179e6678f5cb4b5444 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d626aede5dd49179e6678f5cb4b5444 2024-11-18T20:09:57,601 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d626aede5dd49179e6678f5cb4b5444, entries=150, sequenceid=77, filesize=11.7 K 2024-11-18T20:09:57,602 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c01d05a23f041191706ac16b56e11567 in 914ms, sequenceid=77, compaction requested=false 2024-11-18T20:09:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-18T20:09:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-18T20:09:57,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-18T20:09:57,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-18T20:09:57,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.7260 sec 2024-11-18T20:09:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:57,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:09:57,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:09:57,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181f6de5f8ff18482e93b3448112e64b98_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742190_1366 (size=14594) 2024-11-18T20:09:57,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:57,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181f6de5f8ff18482e93b3448112e64b98_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181f6de5f8ff18482e93b3448112e64b98_c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:57,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/87c5cace22564e0f9c7dc0b8119e7439, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:57,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/87c5cace22564e0f9c7dc0b8119e7439 is 175, key is test_row_0/A:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742191_1367 (size=39549) 2024-11-18T20:09:57,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960657723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960657728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960657842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960657842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960658011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,017 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960658022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,026 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T20:09:58,032 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:09:58,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:09:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-18T20:09:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:09:58,035 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:09:58,035 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:09:58,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:09:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960658032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,040 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:09:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960658051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960658052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/87c5cace22564e0f9c7dc0b8119e7439 2024-11-18T20:09:58,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/fe63a80bf31341ddb08d0682dadb34a8 is 50, key is test_row_0/B:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:58,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742192_1368 (size=12001) 2024-11-18T20:09:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:09:58,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:09:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:09:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:09:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960658362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960658362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:09:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:09:58,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/fe63a80bf31341ddb08d0682dadb34a8 2024-11-18T20:09:58,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/f49750348e9e4144b25e1bd908272b33 is 50, key is test_row_0/C:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:58,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742193_1369 (size=12001) 2024-11-18T20:09:58,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/f49750348e9e4144b25e1bd908272b33 2024-11-18T20:09:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/87c5cace22564e0f9c7dc0b8119e7439 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439 2024-11-18T20:09:58,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439, entries=200, sequenceid=93, filesize=38.6 K 2024-11-18T20:09:58,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/fe63a80bf31341ddb08d0682dadb34a8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/fe63a80bf31341ddb08d0682dadb34a8 2024-11-18T20:09:58,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/fe63a80bf31341ddb08d0682dadb34a8, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:09:58,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/f49750348e9e4144b25e1bd908272b33 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/f49750348e9e4144b25e1bd908272b33 2024-11-18T20:09:58,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/f49750348e9e4144b25e1bd908272b33, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:09:58,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c01d05a23f041191706ac16b56e11567 in 990ms, sequenceid=93, compaction requested=true 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:58,618 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:58,618 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:09:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:58,622 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:58,622 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:58,622 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:09:58,622 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:09:58,622 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,622 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ec5a86d8bbdc47359fc41c2cd90c37b5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=99.2 K 2024-11-18T20:09:58,622 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,622 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ec5a86d8bbdc47359fc41c2cd90c37b5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439] 2024-11-18T20:09:58,622 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,622 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a85f6a3aa36546a2804eadfc9414ddef, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/7540c2a7210e4024a9448dbcd0d23710, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/fe63a80bf31341ddb08d0682dadb34a8] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=35.3 K 2024-11-18T20:09:58,623 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5a86d8bbdc47359fc41c2cd90c37b5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960594974 2024-11-18T20:09:58,623 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a85f6a3aa36546a2804eadfc9414ddef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960594974 2024-11-18T20:09:58,626 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7540c2a7210e4024a9448dbcd0d23710, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731960595716 2024-11-18T20:09:58,626 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1243b4cb74a643cbababa1f2cda17b09, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731960595716 2024-11-18T20:09:58,627 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting fe63a80bf31341ddb08d0682dadb34a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960596967 2024-11-18T20:09:58,627 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c5cace22564e0f9c7dc0b8119e7439, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960596967 2024-11-18T20:09:58,637 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:58,638 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/0cc6c006a94145cf8f15fd85a9267a8f is 50, key is test_row_0/B:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:58,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:09:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:58,652 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:09:58,660 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:09:58,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742194_1370 (size=12207) 2024-11-18T20:09:58,700 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ffeedc1143104cee9f7a4f70aaad0889_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:58,703 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ffeedc1143104cee9f7a4f70aaad0889_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:58,703 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ffeedc1143104cee9f7a4f70aaad0889_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188be103c461fa414bacf4f5e069a04e02_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960597727/Put/seqid=0 2024-11-18T20:09:58,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742195_1371 (size=4469) 2024-11-18T20:09:58,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742196_1372 (size=12154) 2024-11-18T20:09:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:58,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:09:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960658903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:58,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960658914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960659022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:59,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960659022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,095 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/0cc6c006a94145cf8f15fd85a9267a8f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0cc6c006a94145cf8f15fd85a9267a8f 2024-11-18T20:09:59,100 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into 0cc6c006a94145cf8f15fd85a9267a8f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:59,100 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:59,100 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=13, startTime=1731960598618; duration=0sec 2024-11-18T20:09:59,100 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:09:59,100 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:09:59,100 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:09:59,101 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:09:59,101 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:09:59,101 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:09:59,101 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1c71af87055f44f9a559317d2907e9e3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d626aede5dd49179e6678f5cb4b5444, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/f49750348e9e4144b25e1bd908272b33] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=35.3 K 2024-11-18T20:09:59,102 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c71af87055f44f9a559317d2907e9e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731960594974 2024-11-18T20:09:59,102 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d626aede5dd49179e6678f5cb4b5444, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731960595716 2024-11-18T20:09:59,102 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f49750348e9e4144b25e1bd908272b33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960596967 2024-11-18T20:09:59,111 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#319 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:59,111 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/7ca51dc108e34e7482e031d6f2d11d04 is 175, key is test_row_0/A:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:59,113 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:09:59,114 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/0ff46429feac4a27a05bfa4a332d0064 is 50, key is test_row_0/C:col10/1731960597627/Put/seqid=0 2024-11-18T20:09:59,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742197_1373 (size=31161) 2024-11-18T20:09:59,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742198_1374 (size=12207) 2024-11-18T20:09:59,142 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/0ff46429feac4a27a05bfa4a332d0064 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/0ff46429feac4a27a05bfa4a332d0064 2024-11-18T20:09:59,146 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into 0ff46429feac4a27a05bfa4a332d0064(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:59,146 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:59,146 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=13, startTime=1731960598618; duration=0sec 2024-11-18T20:09:59,146 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:59,146 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:09:59,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:09:59,154 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188be103c461fa414bacf4f5e069a04e02_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188be103c461fa414bacf4f5e069a04e02_c01d05a23f041191706ac16b56e11567 2024-11-18T20:09:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/2080bcea30574ca787333bd1f6d4f7ac, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:09:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/2080bcea30574ca787333bd1f6d4f7ac is 175, key is test_row_0/A:col10/1731960597727/Put/seqid=0 2024-11-18T20:09:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:09:59,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742199_1375 (size=30955) 2024-11-18T20:09:59,183 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/2080bcea30574ca787333bd1f6d4f7ac 2024-11-18T20:09:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/06cbebce9b3a4e3db5286d903a63bcf9 is 50, key is test_row_0/B:col10/1731960597727/Put/seqid=0 2024-11-18T20:09:59,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960659243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960659243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742200_1376 (size=12001) 2024-11-18T20:09:59,534 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/7ca51dc108e34e7482e031d6f2d11d04 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7ca51dc108e34e7482e031d6f2d11d04 2024-11-18T20:09:59,540 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into 7ca51dc108e34e7482e031d6f2d11d04(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:09:59,540 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:09:59,541 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=13, startTime=1731960598618; duration=0sec 2024-11-18T20:09:59,541 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:09:59,541 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:09:59,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960659562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:59,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:09:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960659562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:09:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:09:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:09:59,655 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/06cbebce9b3a4e3db5286d903a63bcf9 2024-11-18T20:09:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/708e6ba2a94b4525a8f9974e22d2b8bb is 50, key is test_row_0/C:col10/1731960597727/Put/seqid=0 2024-11-18T20:09:59,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742201_1377 (size=12001) 2024-11-18T20:10:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960660072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960660072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:00,082 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/708e6ba2a94b4525a8f9974e22d2b8bb 2024-11-18T20:10:00,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/2080bcea30574ca787333bd1f6d4f7ac as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac 2024-11-18T20:10:00,099 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac, entries=150, sequenceid=116, filesize=30.2 K 2024-11-18T20:10:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/06cbebce9b3a4e3db5286d903a63bcf9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/06cbebce9b3a4e3db5286d903a63bcf9 2024-11-18T20:10:00,104 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/06cbebce9b3a4e3db5286d903a63bcf9, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T20:10:00,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/708e6ba2a94b4525a8f9974e22d2b8bb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/708e6ba2a94b4525a8f9974e22d2b8bb 2024-11-18T20:10:00,110 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/708e6ba2a94b4525a8f9974e22d2b8bb, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T20:10:00,116 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c01d05a23f041191706ac16b56e11567 in 1463ms, sequenceid=116, compaction requested=false 2024-11-18T20:10:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-18T20:10:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-18T20:10:00,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-18T20:10:00,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0830 sec 2024-11-18T20:10:00,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.0910 sec 2024-11-18T20:10:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:10:00,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:00,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-18T20:10:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:10:00,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:00,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:00,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:10:00,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:10:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:00,347 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:10:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d5edd615a04c4070938e0bfb7ad5be3a_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960598913/Put/seqid=0 2024-11-18T20:10:00,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742202_1378 (size=12154) 2024-11-18T20:10:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:10:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:00,779 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d5edd615a04c4070938e0bfb7ad5be3a_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5edd615a04c4070938e0bfb7ad5be3a_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:00,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/8c216943b69b4893b7aa3b0df1bd47d3, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:00,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/8c216943b69b4893b7aa3b0df1bd47d3 is 175, key is test_row_0/A:col10/1731960598913/Put/seqid=0 2024-11-18T20:10:00,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742203_1379 (size=30955) 2024-11-18T20:10:00,785 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/8c216943b69b4893b7aa3b0df1bd47d3 2024-11-18T20:10:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/51a9b1e4e505402ca674f8e18835e26c is 50, key is test_row_0/B:col10/1731960598913/Put/seqid=0 2024-11-18T20:10:00,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742204_1380 (size=12001) 2024-11-18T20:10:00,796 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/51a9b1e4e505402ca674f8e18835e26c 2024-11-18T20:10:00,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/748fd90386004dd3bf287dab860e27c5 is 50, key is test_row_0/C:col10/1731960598913/Put/seqid=0 2024-11-18T20:10:00,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742205_1381 (size=12001) 2024-11-18T20:10:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:10:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:01,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960661188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960661193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,217 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/748fd90386004dd3bf287dab860e27c5 2024-11-18T20:10:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/8c216943b69b4893b7aa3b0df1bd47d3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3 2024-11-18T20:10:01,258 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3, entries=150, sequenceid=132, filesize=30.2 K 2024-11-18T20:10:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/51a9b1e4e505402ca674f8e18835e26c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/51a9b1e4e505402ca674f8e18835e26c 2024-11-18T20:10:01,270 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/51a9b1e4e505402ca674f8e18835e26c, entries=150, sequenceid=132, filesize=11.7 K 2024-11-18T20:10:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/748fd90386004dd3bf287dab860e27c5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/748fd90386004dd3bf287dab860e27c5 2024-11-18T20:10:01,288 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/748fd90386004dd3bf287dab860e27c5, entries=150, sequenceid=132, filesize=11.7 K 2024-11-18T20:10:01,294 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c01d05a23f041191706ac16b56e11567 in 946ms, sequenceid=132, compaction requested=true 2024-11-18T20:10:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-18T20:10:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-18T20:10:01,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-18T20:10:01,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1010 sec 2024-11-18T20:10:01,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1050 sec 2024-11-18T20:10:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:01,312 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:01,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:10:01,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180868fc2a83864cfcb743dc03da75cb6c_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:10:01,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:01,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-18T20:10:01,345 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:01,346 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:01,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:01,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742206_1382 (size=17284) 2024-11-18T20:10:01,366 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:01,371 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180868fc2a83864cfcb743dc03da75cb6c_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180868fc2a83864cfcb743dc03da75cb6c_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:01,372 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/20a400ff64634b1cba5e0dfcfeea95fe, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:01,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/20a400ff64634b1cba5e0dfcfeea95fe is 175, key is test_row_0/A:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:01,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960661367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742207_1383 (size=48389) 2024-11-18T20:10:01,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/20a400ff64634b1cba5e0dfcfeea95fe 2024-11-18T20:10:01,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960661375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/1ec66b377d0d4beebe7be92969cc641f is 50, key is test_row_0/B:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742208_1384 (size=12151) 2024-11-18T20:10:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:01,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960661482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960661492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:10:01,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:01,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:10:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:01,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960661692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960661702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:01,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:10:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:01,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/1ec66b377d0d4beebe7be92969cc641f 2024-11-18T20:10:01,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/6e33984051cc46afa4bf448e1e88eb83 is 50, key is test_row_0/C:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:01,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742209_1385 (size=12151) 2024-11-18T20:10:01,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/6e33984051cc46afa4bf448e1e88eb83 2024-11-18T20:10:01,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/20a400ff64634b1cba5e0dfcfeea95fe as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe 2024-11-18T20:10:01,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe, entries=250, sequenceid=157, filesize=47.3 K 2024-11-18T20:10:01,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/1ec66b377d0d4beebe7be92969cc641f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1ec66b377d0d4beebe7be92969cc641f 2024-11-18T20:10:01,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1ec66b377d0d4beebe7be92969cc641f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:10:01,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/6e33984051cc46afa4bf448e1e88eb83 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6e33984051cc46afa4bf448e1e88eb83 2024-11-18T20:10:01,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6e33984051cc46afa4bf448e1e88eb83, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:10:01,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c01d05a23f041191706ac16b56e11567 in 634ms, sequenceid=157, compaction requested=true 2024-11-18T20:10:01,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:01,946 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:01,947 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:01,947 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:01,948 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:10:01,948 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,948 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7ca51dc108e34e7482e031d6f2d11d04, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=138.1 K 2024-11-18T20:10:01,948 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,948 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7ca51dc108e34e7482e031d6f2d11d04, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe] 2024-11-18T20:10:01,948 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca51dc108e34e7482e031d6f2d11d04, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960596967 2024-11-18T20:10:01,948 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:01,948 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:10:01,949 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,949 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0cc6c006a94145cf8f15fd85a9267a8f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/06cbebce9b3a4e3db5286d903a63bcf9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/51a9b1e4e505402ca674f8e18835e26c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1ec66b377d0d4beebe7be92969cc641f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=47.2 K 2024-11-18T20:10:01,949 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2080bcea30574ca787333bd1f6d4f7ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731960597716 2024-11-18T20:10:01,949 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc6c006a94145cf8f15fd85a9267a8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960596967 2024-11-18T20:10:01,949 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c216943b69b4893b7aa3b0df1bd47d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960598891 2024-11-18T20:10:01,949 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 06cbebce9b3a4e3db5286d903a63bcf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731960597716 2024-11-18T20:10:01,949 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a400ff64634b1cba5e0dfcfeea95fe, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960601172 2024-11-18T20:10:01,950 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a9b1e4e505402ca674f8e18835e26c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960598891 2024-11-18T20:10:01,950 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec66b377d0d4beebe7be92969cc641f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960601172 2024-11-18T20:10:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:01,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:10:01,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:01,988 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:10:01,996 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:02,007 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:02,007 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3ec9658df6064da198976b6e6b7abf99 is 50, key is test_row_0/B:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:02,011 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ddbc90aac4194c12a496ff9aa2bbbfd4_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:02,014 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ddbc90aac4194c12a496ff9aa2bbbfd4_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:02,014 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ddbc90aac4194c12a496ff9aa2bbbfd4_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:02,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118dd765c445296459db2e3bf117d8d5a5c_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960601344/Put/seqid=0 2024-11-18T20:10:02,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742211_1387 (size=4469) 2024-11-18T20:10:02,055 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#330 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:02,055 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/42b3cfafe8c2497b8a5bdd42c6a5d80b is 175, key is test_row_0/A:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:02,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742210_1386 (size=12493) 2024-11-18T20:10:02,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742212_1388 (size=12304) 2024-11-18T20:10:02,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742213_1389 (size=31447) 2024-11-18T20:10:02,090 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/42b3cfafe8c2497b8a5bdd42c6a5d80b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/42b3cfafe8c2497b8a5bdd42c6a5d80b 2024-11-18T20:10:02,096 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into 42b3cfafe8c2497b8a5bdd42c6a5d80b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:02,096 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:02,096 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=12, startTime=1731960601946; duration=0sec 2024-11-18T20:10:02,096 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:02,096 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:10:02,096 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:02,097 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:02,098 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:10:02,098 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:02,098 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/0ff46429feac4a27a05bfa4a332d0064, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/708e6ba2a94b4525a8f9974e22d2b8bb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/748fd90386004dd3bf287dab860e27c5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6e33984051cc46afa4bf448e1e88eb83] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=47.2 K 2024-11-18T20:10:02,098 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff46429feac4a27a05bfa4a332d0064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960596967 2024-11-18T20:10:02,099 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708e6ba2a94b4525a8f9974e22d2b8bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731960597716 2024-11-18T20:10:02,099 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748fd90386004dd3bf287dab860e27c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960598891 2024-11-18T20:10:02,099 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e33984051cc46afa4bf448e1e88eb83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960601172 2024-11-18T20:10:02,118 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:02,118 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/404adefbe4684ab5927ab74fb147338a is 50, key is test_row_0/C:col10/1731960601172/Put/seqid=0 2024-11-18T20:10:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960662111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960662112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960662113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960662125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960662127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742214_1390 (size=12493) 2024-11-18T20:10:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960662232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960662233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960662233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960662242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960662255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960662442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960662452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960662452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:02,484 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3ec9658df6064da198976b6e6b7abf99 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3ec9658df6064da198976b6e6b7abf99 2024-11-18T20:10:02,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960662480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960662480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,488 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118dd765c445296459db2e3bf117d8d5a5c_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dd765c445296459db2e3bf117d8d5a5c_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5f35717c73ce434db652fc5b61397d0a, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5f35717c73ce434db652fc5b61397d0a is 175, key is test_row_0/A:col10/1731960601344/Put/seqid=0 2024-11-18T20:10:02,498 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into 3ec9658df6064da198976b6e6b7abf99(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:02,498 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:02,498 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=12, startTime=1731960601947; duration=0sec 2024-11-18T20:10:02,498 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:02,498 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:10:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742215_1391 (size=31105) 2024-11-18T20:10:02,553 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/404adefbe4684ab5927ab74fb147338a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/404adefbe4684ab5927ab74fb147338a 2024-11-18T20:10:02,557 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into 404adefbe4684ab5927ab74fb147338a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:02,557 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:02,557 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=12, startTime=1731960601947; duration=0sec 2024-11-18T20:10:02,557 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:02,557 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:10:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960662752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960662772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960662781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960662791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960662792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:02,913 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5f35717c73ce434db652fc5b61397d0a 2024-11-18T20:10:02,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/f0914e9987f94c56a399c975ea4da34f is 50, key is test_row_0/B:col10/1731960601344/Put/seqid=0 2024-11-18T20:10:02,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742216_1392 (size=12151) 2024-11-18T20:10:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960663262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960663281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:03,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960663292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:03,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960663302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:03,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960663302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:03,322 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/f0914e9987f94c56a399c975ea4da34f 2024-11-18T20:10:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/4d73cd9cbd3148ec94dfbc27c3fc35f4 is 50, key is test_row_0/C:col10/1731960601344/Put/seqid=0 2024-11-18T20:10:03,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742217_1393 (size=12151) 2024-11-18T20:10:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:03,731 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/4d73cd9cbd3148ec94dfbc27c3fc35f4 2024-11-18T20:10:03,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/5f35717c73ce434db652fc5b61397d0a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a 2024-11-18T20:10:03,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a, entries=150, sequenceid=168, filesize=30.4 K 2024-11-18T20:10:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/f0914e9987f94c56a399c975ea4da34f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/f0914e9987f94c56a399c975ea4da34f 2024-11-18T20:10:03,750 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/f0914e9987f94c56a399c975ea4da34f, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T20:10:03,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/4d73cd9cbd3148ec94dfbc27c3fc35f4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d73cd9cbd3148ec94dfbc27c3fc35f4 2024-11-18T20:10:03,758 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d73cd9cbd3148ec94dfbc27c3fc35f4, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T20:10:03,763 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c01d05a23f041191706ac16b56e11567 in 1775ms, sequenceid=168, compaction requested=false 2024-11-18T20:10:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-18T20:10:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-18T20:10:03,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-18T20:10:03,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4190 sec 2024-11-18T20:10:03,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.4260 sec 2024-11-18T20:10:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:04,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:10:04,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ba1cc79061c1461586faca403f32adbe_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:04,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742218_1394 (size=14794) 2024-11-18T20:10:04,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960664318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960664318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960664319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960664321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960664322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,331 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960664432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960664432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960664442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960664442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960664642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960664642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960664652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960664652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:04,704 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ba1cc79061c1461586faca403f32adbe_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ba1cc79061c1461586faca403f32adbe_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:04,704 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/d5122d7df7624ddd8bb73c46631a0255, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:04,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/d5122d7df7624ddd8bb73c46631a0255 is 175, key is test_row_0/A:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:04,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742219_1395 (size=39749) 2024-11-18T20:10:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960664952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960664952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960664962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:04,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960664962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:05,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/d5122d7df7624ddd8bb73c46631a0255 2024-11-18T20:10:05,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/eb36bb76f65848c5abee7662b7bb7867 is 50, key is test_row_0/B:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:05,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742220_1396 (size=12151) 2024-11-18T20:10:05,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960665472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:05,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960665473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960665473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960665473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:10:05,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:05,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-18T20:10:05,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:05,513 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:05,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:10:05,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/eb36bb76f65848c5abee7662b7bb7867 2024-11-18T20:10:05,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b288c42f3f234543ad1c7f9e97180718 is 50, key is test_row_0/C:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:05,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742221_1397 (size=12151) 2024-11-18T20:10:05,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b288c42f3f234543ad1c7f9e97180718 2024-11-18T20:10:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:10:05,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/d5122d7df7624ddd8bb73c46631a0255 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255 2024-11-18T20:10:05,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255, entries=200, sequenceid=197, filesize=38.8 K 2024-11-18T20:10:05,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/eb36bb76f65848c5abee7662b7bb7867 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/eb36bb76f65848c5abee7662b7bb7867 2024-11-18T20:10:05,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:10:05,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:05,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:05,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:05,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:05,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:05,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/eb36bb76f65848c5abee7662b7bb7867, entries=150, sequenceid=197, filesize=11.9 K 2024-11-18T20:10:05,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b288c42f3f234543ad1c7f9e97180718 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b288c42f3f234543ad1c7f9e97180718 2024-11-18T20:10:05,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b288c42f3f234543ad1c7f9e97180718, entries=150, sequenceid=197, filesize=11.9 K 2024-11-18T20:10:05,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c01d05a23f041191706ac16b56e11567 in 1399ms, sequenceid=197, compaction requested=true 2024-11-18T20:10:05,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:05,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:05,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:05,686 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:05,687 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:05,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:05,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:05,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:05,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:05,687 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:05,687 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:05,687 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:10:05,687 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:10:05,688 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:05,688 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:05,688 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3ec9658df6064da198976b6e6b7abf99, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/f0914e9987f94c56a399c975ea4da34f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/eb36bb76f65848c5abee7662b7bb7867] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=35.9 K 2024-11-18T20:10:05,688 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/42b3cfafe8c2497b8a5bdd42c6a5d80b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=99.9 K 2024-11-18T20:10:05,688 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:05,688 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/42b3cfafe8c2497b8a5bdd42c6a5d80b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255] 2024-11-18T20:10:05,688 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec9658df6064da198976b6e6b7abf99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960601172 2024-11-18T20:10:05,688 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b3cfafe8c2497b8a5bdd42c6a5d80b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960601172 2024-11-18T20:10:05,688 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f0914e9987f94c56a399c975ea4da34f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731960601344 2024-11-18T20:10:05,688 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f35717c73ce434db652fc5b61397d0a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731960601344 2024-11-18T20:10:05,689 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting eb36bb76f65848c5abee7662b7bb7867, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731960602089 2024-11-18T20:10:05,689 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5122d7df7624ddd8bb73c46631a0255, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731960602089 2024-11-18T20:10:05,708 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:05,710 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411184ec34f8244554b4584fc931fc23ea0e7_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:05,712 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#339 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:05,712 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411184ec34f8244554b4584fc931fc23ea0e7_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:05,712 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184ec34f8244554b4584fc931fc23ea0e7_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:05,712 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/1240aa908cd349c389d97024385c134b is 50, key is test_row_0/B:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:05,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742222_1398 (size=4469) 2024-11-18T20:10:05,744 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#340 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:05,745 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/299eb34f51ce40bfb05571dea9ae1df2 is 175, key is test_row_0/A:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:05,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742223_1399 (size=12595) 2024-11-18T20:10:05,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742224_1400 (size=31549) 2024-11-18T20:10:05,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:10:05,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:05,831 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-18T20:10:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:10:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184d24440b196a474b8c7d99ac4584638d_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960604318/Put/seqid=0 2024-11-18T20:10:05,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742225_1401 (size=12304) 2024-11-18T20:10:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:10:06,222 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/1240aa908cd349c389d97024385c134b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1240aa908cd349c389d97024385c134b 2024-11-18T20:10:06,227 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/299eb34f51ce40bfb05571dea9ae1df2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/299eb34f51ce40bfb05571dea9ae1df2 2024-11-18T20:10:06,247 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into 1240aa908cd349c389d97024385c134b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:06,247 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:06,247 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=13, startTime=1731960605686; duration=0sec 2024-11-18T20:10:06,247 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:06,247 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:10:06,247 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:06,248 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into 299eb34f51ce40bfb05571dea9ae1df2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:06,248 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:06,248 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=13, startTime=1731960605686; duration=0sec 2024-11-18T20:10:06,248 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:06,249 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:10:06,250 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:06,250 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:10:06,250 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:06,250 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/404adefbe4684ab5927ab74fb147338a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d73cd9cbd3148ec94dfbc27c3fc35f4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b288c42f3f234543ad1c7f9e97180718] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=35.9 K 2024-11-18T20:10:06,251 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 404adefbe4684ab5927ab74fb147338a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960601172 2024-11-18T20:10:06,251 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d73cd9cbd3148ec94dfbc27c3fc35f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731960601344 2024-11-18T20:10:06,251 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b288c42f3f234543ad1c7f9e97180718, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731960602089 2024-11-18T20:10:06,281 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:06,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:06,282 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/c3fea0084bad4c5481984754dc4388fb is 50, key is test_row_0/C:col10/1731960602124/Put/seqid=0 2024-11-18T20:10:06,288 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184d24440b196a474b8c7d99ac4584638d_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184d24440b196a474b8c7d99ac4584638d_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ae8837829f984dd1ab668c6d49957d46, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ae8837829f984dd1ab668c6d49957d46 is 175, key is test_row_0/A:col10/1731960604318/Put/seqid=0 2024-11-18T20:10:06,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742226_1402 (size=12595) 2024-11-18T20:10:06,340 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/c3fea0084bad4c5481984754dc4388fb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/c3fea0084bad4c5481984754dc4388fb 2024-11-18T20:10:06,348 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into c3fea0084bad4c5481984754dc4388fb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:06,348 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:06,348 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=13, startTime=1731960605687; duration=0sec 2024-11-18T20:10:06,348 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:06,348 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:10:06,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742227_1403 (size=31105) 2024-11-18T20:10:06,351 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ae8837829f984dd1ab668c6d49957d46 2024-11-18T20:10:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:06,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/15688e4cdb54489fbfbe5b2e346ea0bf is 50, key is test_row_0/B:col10/1731960604318/Put/seqid=0 2024-11-18T20:10:06,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742228_1404 (size=12151) 2024-11-18T20:10:06,373 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/15688e4cdb54489fbfbe5b2e346ea0bf 2024-11-18T20:10:06,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f is 50, key is test_row_0/C:col10/1731960604318/Put/seqid=0 2024-11-18T20:10:06,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742229_1405 (size=12151) 2024-11-18T20:10:06,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960666482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,491 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960666488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960666489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960666491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960666491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,499 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960666602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960666607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960666611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:10:06,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f 2024-11-18T20:10:06,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ae8837829f984dd1ab668c6d49957d46 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46 2024-11-18T20:10:06,813 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46, entries=150, sequenceid=207, filesize=30.4 K 2024-11-18T20:10:06,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/15688e4cdb54489fbfbe5b2e346ea0bf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/15688e4cdb54489fbfbe5b2e346ea0bf 2024-11-18T20:10:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960666812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,823 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/15688e4cdb54489fbfbe5b2e346ea0bf, entries=150, sequenceid=207, filesize=11.9 K 2024-11-18T20:10:06,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f 2024-11-18T20:10:06,829 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f, entries=150, sequenceid=207, filesize=11.9 K 2024-11-18T20:10:06,829 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for c01d05a23f041191706ac16b56e11567 in 998ms, sequenceid=207, compaction requested=false 2024-11-18T20:10:06,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:06,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:06,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-18T20:10:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-18T20:10:06,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:06,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:10:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:06,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-18T20:10:06,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3180 sec 2024-11-18T20:10:06,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3250 sec 2024-11-18T20:10:06,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118dc4e28706451435889977fb23cc3d646_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742230_1406 (size=14794) 2024-11-18T20:10:06,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960666849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960666851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960666964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960666965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960667123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960667172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960667182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:07,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118dc4e28706451435889977fb23cc3d646_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dc4e28706451435889977fb23cc3d646_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:07,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a2a4732a6ab847528466b4b0078e2214, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:07,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a2a4732a6ab847528466b4b0078e2214 is 175, key is test_row_0/A:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:07,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742231_1407 (size=39749) 2024-11-18T20:10:07,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a2a4732a6ab847528466b4b0078e2214 2024-11-18T20:10:07,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3e90984767134285b4cb2c7188c091cd is 50, key is test_row_0/B:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:07,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742232_1408 (size=12151) 2024-11-18T20:10:07,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3e90984767134285b4cb2c7188c091cd 2024-11-18T20:10:07,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/316aae6f6bec4e169e4fde4fb6682f64 is 50, key is test_row_0/C:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:07,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742233_1409 (size=12151) 2024-11-18T20:10:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960667492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960667502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960667642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:07,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:10:07,663 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:07,663 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-18T20:10:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:10:07,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:07,665 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:07,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:07,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/316aae6f6bec4e169e4fde4fb6682f64 2024-11-18T20:10:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a2a4732a6ab847528466b4b0078e2214 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214 2024-11-18T20:10:07,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214, entries=200, sequenceid=237, filesize=38.8 K 2024-11-18T20:10:07,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3e90984767134285b4cb2c7188c091cd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e90984767134285b4cb2c7188c091cd 2024-11-18T20:10:07,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e90984767134285b4cb2c7188c091cd, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T20:10:07,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/316aae6f6bec4e169e4fde4fb6682f64 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/316aae6f6bec4e169e4fde4fb6682f64 2024-11-18T20:10:07,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/316aae6f6bec4e169e4fde4fb6682f64, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T20:10:07,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for c01d05a23f041191706ac16b56e11567 in 907ms, sequenceid=237, compaction requested=true 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:07,738 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:07,738 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:07,739 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:07,739 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:07,739 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:10:07,739 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:10:07,739 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:07,739 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:07,739 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/299eb34f51ce40bfb05571dea9ae1df2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=100.0 K 2024-11-18T20:10:07,739 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1240aa908cd349c389d97024385c134b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/15688e4cdb54489fbfbe5b2e346ea0bf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e90984767134285b4cb2c7188c091cd] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=36.0 K 2024-11-18T20:10:07,739 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:07,739 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/299eb34f51ce40bfb05571dea9ae1df2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214] 2024-11-18T20:10:07,739 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1240aa908cd349c389d97024385c134b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731960602089 2024-11-18T20:10:07,740 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 15688e4cdb54489fbfbe5b2e346ea0bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731960604316 2024-11-18T20:10:07,740 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299eb34f51ce40bfb05571dea9ae1df2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731960602089 2024-11-18T20:10:07,740 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e90984767134285b4cb2c7188c091cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960606486 2024-11-18T20:10:07,740 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae8837829f984dd1ab668c6d49957d46, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731960604316 2024-11-18T20:10:07,740 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a4732a6ab847528466b4b0078e2214, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960606486 2024-11-18T20:10:07,752 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:07,754 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:07,755 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a2365486ddfd40648d3068dc8a3fd897 is 50, key is test_row_0/B:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:07,768 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118bb9bcd2275b04f70ad41daa4e098d45a_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:07,770 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118bb9bcd2275b04f70ad41daa4e098d45a_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:07,770 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bb9bcd2275b04f70ad41daa4e098d45a_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:10:07,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742234_1410 (size=12697) 2024-11-18T20:10:07,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742235_1411 (size=4469) 2024-11-18T20:10:07,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:10:07,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:07,817 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-18T20:10:07,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182cd98e1966d241d8b953b6da7bdeb165_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960606850/Put/seqid=0 2024-11-18T20:10:07,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742236_1412 (size=12304) 2024-11-18T20:10:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:10:08,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960668137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960668139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,197 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a2365486ddfd40648d3068dc8a3fd897 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a2365486ddfd40648d3068dc8a3fd897 2024-11-18T20:10:08,198 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#348 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:08,198 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ae65911c5db64665aa22afefb817b446 is 175, key is test_row_0/A:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:08,204 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into a2365486ddfd40648d3068dc8a3fd897(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:08,205 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:08,205 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=13, startTime=1731960607738; duration=0sec 2024-11-18T20:10:08,205 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:08,205 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:10:08,205 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:08,206 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:08,206 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:10:08,207 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:08,207 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/c3fea0084bad4c5481984754dc4388fb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/316aae6f6bec4e169e4fde4fb6682f64] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=36.0 K 2024-11-18T20:10:08,207 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting c3fea0084bad4c5481984754dc4388fb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731960602089 2024-11-18T20:10:08,208 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b09fa42d7e7b4ef2ab6c7ac121d3dc6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731960604316 2024-11-18T20:10:08,208 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 316aae6f6bec4e169e4fde4fb6682f64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960606486 2024-11-18T20:10:08,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:08,240 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182cd98e1966d241d8b953b6da7bdeb165_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182cd98e1966d241d8b953b6da7bdeb165_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:08,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/90a08a6bc23b4221bc9fb926d1780fd0, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:08,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/90a08a6bc23b4221bc9fb926d1780fd0 is 175, key is test_row_0/A:col10/1731960606850/Put/seqid=0 2024-11-18T20:10:08,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742237_1413 (size=31651) 2024-11-18T20:10:08,260 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#351 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:08,261 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b847ff9df2084e15876f0fcda97a4254 is 50, key is test_row_0/C:col10/1731960606830/Put/seqid=0 2024-11-18T20:10:08,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960668252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960668252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,268 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/ae65911c5db64665aa22afefb817b446 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae65911c5db64665aa22afefb817b446 2024-11-18T20:10:08,274 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into ae65911c5db64665aa22afefb817b446(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:08,274 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:08,274 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=13, startTime=1731960607738; duration=0sec 2024-11-18T20:10:08,274 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:08,274 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:10:08,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742238_1414 (size=31105) 2024-11-18T20:10:08,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742239_1415 (size=12697) 2024-11-18T20:10:08,291 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/90a08a6bc23b4221bc9fb926d1780fd0 2024-11-18T20:10:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:10:08,302 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b847ff9df2084e15876f0fcda97a4254 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b847ff9df2084e15876f0fcda97a4254 2024-11-18T20:10:08,316 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into b847ff9df2084e15876f0fcda97a4254(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:08,316 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:08,316 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=13, startTime=1731960607738; duration=0sec 2024-11-18T20:10:08,316 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:08,316 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:10:08,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3e8e065d8ee74229927f862ad5b00ac9 is 50, key is test_row_0/B:col10/1731960606850/Put/seqid=0 2024-11-18T20:10:08,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742240_1416 (size=12151) 2024-11-18T20:10:08,343 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3e8e065d8ee74229927f862ad5b00ac9 2024-11-18T20:10:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b6644d0c20c04b4d87f806f98ef162ad is 50, key is test_row_0/C:col10/1731960606850/Put/seqid=0 2024-11-18T20:10:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742241_1417 (size=12151) 2024-11-18T20:10:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960668472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960668472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960668511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960668511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,519 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,519 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960668655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,661 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,767 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b6644d0c20c04b4d87f806f98ef162ad 2024-11-18T20:10:08,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/90a08a6bc23b4221bc9fb926d1780fd0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0 2024-11-18T20:10:08,774 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0, entries=150, sequenceid=246, filesize=30.4 K 2024-11-18T20:10:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/3e8e065d8ee74229927f862ad5b00ac9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e8e065d8ee74229927f862ad5b00ac9 2024-11-18T20:10:08,778 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e8e065d8ee74229927f862ad5b00ac9, entries=150, sequenceid=246, filesize=11.9 K 2024-11-18T20:10:08,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/b6644d0c20c04b4d87f806f98ef162ad as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b6644d0c20c04b4d87f806f98ef162ad 2024-11-18T20:10:08,781 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b6644d0c20c04b4d87f806f98ef162ad, entries=150, sequenceid=246, filesize=11.9 K 2024-11-18T20:10:08,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for c01d05a23f041191706ac16b56e11567 in 965ms, sequenceid=246, compaction requested=false 2024-11-18T20:10:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-18T20:10:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-18T20:10:08,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-18T20:10:08,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1180 sec 2024-11-18T20:10:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:08,786 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:08,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.1220 sec 2024-11-18T20:10:08,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-18T20:10:08,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a7ffa4855de14b02864450f0b07cee52_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:08,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742242_1418 (size=12454) 2024-11-18T20:10:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:10:08,802 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:08,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-18T20:10:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:10:08,808 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:08,808 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:08,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960668808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960668810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:10:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960668922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960668922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:08,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:08,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:08,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:08,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:08,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:08,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:08,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:10:09,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960669132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960669132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:09,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:09,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a7ffa4855de14b02864450f0b07cee52_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a7ffa4855de14b02864450f0b07cee52_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:09,204 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f782be5bfc4e4e309f927c8d84455f20, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:09,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f782be5bfc4e4e309f927c8d84455f20 is 175, key is test_row_0/A:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:09,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742243_1419 (size=31255) 2024-11-18T20:10:09,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f782be5bfc4e4e309f927c8d84455f20 2024-11-18T20:10:09,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/822c54c7d1644e859d24eb26688ff595 is 50, key is test_row_0/B:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742244_1420 (size=12301) 2024-11-18T20:10:09,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:10:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960669442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960669442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:09,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/822c54c7d1644e859d24eb26688ff595 2024-11-18T20:10:09,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/341e1e3c4adf4b7d9d7f9ed71208edcb is 50, key is test_row_0/C:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:09,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742245_1421 (size=12301) 2024-11-18T20:10:09,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/341e1e3c4adf4b7d9d7f9ed71208edcb 2024-11-18T20:10:09,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:09,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f782be5bfc4e4e309f927c8d84455f20 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20 2024-11-18T20:10:09,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20, entries=150, sequenceid=277, filesize=30.5 K 2024-11-18T20:10:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/822c54c7d1644e859d24eb26688ff595 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/822c54c7d1644e859d24eb26688ff595 2024-11-18T20:10:09,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/822c54c7d1644e859d24eb26688ff595, entries=150, sequenceid=277, filesize=12.0 K 2024-11-18T20:10:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/341e1e3c4adf4b7d9d7f9ed71208edcb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/341e1e3c4adf4b7d9d7f9ed71208edcb 2024-11-18T20:10:09,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/341e1e3c4adf4b7d9d7f9ed71208edcb, entries=150, sequenceid=277, filesize=12.0 K 2024-11-18T20:10:09,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for c01d05a23f041191706ac16b56e11567 in 1029ms, sequenceid=277, compaction requested=true 2024-11-18T20:10:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:09,815 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:09,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:09,816 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:09,817 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:10:09,817 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,817 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae65911c5db64665aa22afefb817b446, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=91.8 K 2024-11-18T20:10:09,817 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,817 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae65911c5db64665aa22afefb817b446, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20] 2024-11-18T20:10:09,817 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ae65911c5db64665aa22afefb817b446, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960606486 2024-11-18T20:10:09,818 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a08a6bc23b4221bc9fb926d1780fd0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731960606840 2024-11-18T20:10:09,818 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:09,818 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting f782be5bfc4e4e309f927c8d84455f20, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731960608135 2024-11-18T20:10:09,819 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:09,819 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:10:09,819 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,820 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a2365486ddfd40648d3068dc8a3fd897, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e8e065d8ee74229927f862ad5b00ac9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/822c54c7d1644e859d24eb26688ff595] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=36.3 K 2024-11-18T20:10:09,820 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2365486ddfd40648d3068dc8a3fd897, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960606486 2024-11-18T20:10:09,821 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8e065d8ee74229927f862ad5b00ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731960606840 2024-11-18T20:10:09,821 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 822c54c7d1644e859d24eb26688ff595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731960608135 2024-11-18T20:10:09,828 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:09,838 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:09,838 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/69488f98e04f4055ab0d51b272913f29 is 50, key is test_row_0/B:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:09,846 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118e18e148d88c2439eb22f3d1479e8e259_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:09,848 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118e18e148d88c2439eb22f3d1479e8e259_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:09,848 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e18e148d88c2439eb22f3d1479e8e259_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:09,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742246_1422 (size=12949) 2024-11-18T20:10:09,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:10:09,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,901 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:10:09,910 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/69488f98e04f4055ab0d51b272913f29 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/69488f98e04f4055ab0d51b272913f29 2024-11-18T20:10:09,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188338fa10cdba4913bd9e89c0c7a020ef_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960608788/Put/seqid=0 2024-11-18T20:10:09,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742247_1423 (size=4469) 2024-11-18T20:10:09,923 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into 69488f98e04f4055ab0d51b272913f29(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:09,923 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:09,923 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=13, startTime=1731960609815; duration=0sec 2024-11-18T20:10:09,923 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:09,923 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:10:09,923 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:09,925 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:09,925 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:10:09,925 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:09,925 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b847ff9df2084e15876f0fcda97a4254, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b6644d0c20c04b4d87f806f98ef162ad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/341e1e3c4adf4b7d9d7f9ed71208edcb] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=36.3 K 2024-11-18T20:10:09,925 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b847ff9df2084e15876f0fcda97a4254, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731960606486 2024-11-18T20:10:09,926 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6644d0c20c04b4d87f806f98ef162ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731960606840 2024-11-18T20:10:09,926 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 341e1e3c4adf4b7d9d7f9ed71208edcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731960608135 2024-11-18T20:10:09,941 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:09,942 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/8922b2dfca0748c596418db2a30b56e9 is 50, key is test_row_0/C:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:10:09,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742248_1424 (size=12454) 2024-11-18T20:10:09,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:09,957 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188338fa10cdba4913bd9e89c0c7a020ef_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188338fa10cdba4913bd9e89c0c7a020ef_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/6b081fc376bb48b5b475f429a8ec996c, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/6b081fc376bb48b5b475f429a8ec996c is 175, key is test_row_0/A:col10/1731960608788/Put/seqid=0 2024-11-18T20:10:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:09,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742249_1425 (size=12949) 2024-11-18T20:10:10,002 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/8922b2dfca0748c596418db2a30b56e9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8922b2dfca0748c596418db2a30b56e9 2024-11-18T20:10:10,008 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into 8922b2dfca0748c596418db2a30b56e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:10,008 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:10,008 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=13, startTime=1731960609815; duration=0sec 2024-11-18T20:10:10,012 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:10,012 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:10:10,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742250_1426 (size=31255) 2024-11-18T20:10:10,016 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/6b081fc376bb48b5b475f429a8ec996c 2024-11-18T20:10:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/84d10013fa224a6f864f4e8de9a55722 is 50, key is test_row_0/B:col10/1731960608788/Put/seqid=0 2024-11-18T20:10:10,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742251_1427 (size=12301) 2024-11-18T20:10:10,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960670108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960670116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960670222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960670231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,324 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#357 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:10,324 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/0dd3613b42754ec982edcd857efaa7e2 is 175, key is test_row_0/A:col10/1731960608786/Put/seqid=0 2024-11-18T20:10:10,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742252_1428 (size=31903) 2024-11-18T20:10:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960670432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960670442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,464 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/84d10013fa224a6f864f4e8de9a55722 2024-11-18T20:10:10,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/93b495fea1fa4953acc6b7ecb44cbe6e is 50, key is test_row_0/C:col10/1731960608788/Put/seqid=0 2024-11-18T20:10:10,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742253_1429 (size=12301) 2024-11-18T20:10:10,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41302 deadline: 1731960670682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,689 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,737 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/0dd3613b42754ec982edcd857efaa7e2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/0dd3613b42754ec982edcd857efaa7e2 2024-11-18T20:10:10,745 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into 0dd3613b42754ec982edcd857efaa7e2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:10,745 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:10,745 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=13, startTime=1731960609815; duration=0sec 2024-11-18T20:10:10,745 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:10,745 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:10:10,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960670741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960670752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:10,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/93b495fea1fa4953acc6b7ecb44cbe6e 2024-11-18T20:10:10,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/6b081fc376bb48b5b475f429a8ec996c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c 2024-11-18T20:10:10,913 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c, entries=150, sequenceid=285, filesize=30.5 K 2024-11-18T20:10:10,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/84d10013fa224a6f864f4e8de9a55722 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/84d10013fa224a6f864f4e8de9a55722 2024-11-18T20:10:10,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/84d10013fa224a6f864f4e8de9a55722, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T20:10:10,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/93b495fea1fa4953acc6b7ecb44cbe6e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/93b495fea1fa4953acc6b7ecb44cbe6e 2024-11-18T20:10:10,936 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/93b495fea1fa4953acc6b7ecb44cbe6e, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T20:10:10,937 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for c01d05a23f041191706ac16b56e11567 in 1036ms, sequenceid=285, compaction requested=false 2024-11-18T20:10:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-18T20:10:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-18T20:10:10,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-18T20:10:10,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1310 sec 2024-11-18T20:10:10,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1400 sec 2024-11-18T20:10:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:10:10,952 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:10,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-18T20:10:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:10:10,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:10,962 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:10,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:10:11,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-18T20:10:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:11,115 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-18T20:10:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111817d3e8d6a3414ceba4d5c4fa75b74b56_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960610064/Put/seqid=0 2024-11-18T20:10:11,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742254_1430 (size=12454) 2024-11-18T20:10:11,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,177 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111817d3e8d6a3414ceba4d5c4fa75b74b56_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111817d3e8d6a3414ceba4d5c4fa75b74b56_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:11,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:11,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0 is 175, key is test_row_0/A:col10/1731960610064/Put/seqid=0 2024-11-18T20:10:11,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742255_1431 (size=31255) 2024-11-18T20:10:11,229 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0 2024-11-18T20:10:11,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/5549fd203423457cbc4b4fbe63d22da1 is 50, key is test_row_0/B:col10/1731960610064/Put/seqid=0 2024-11-18T20:10:11,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742256_1432 (size=12301) 2024-11-18T20:10:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:11,263 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/5549fd203423457cbc4b4fbe63d22da1 2024-11-18T20:10:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/6973ae34b6d44faebde65df1e45ec969 is 50, key is test_row_0/C:col10/1731960610064/Put/seqid=0 2024-11-18T20:10:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:10:11,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742257_1433 (size=12301) 2024-11-18T20:10:11,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960671275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:11,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960671279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:11,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960671392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960671392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:10:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960671602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:11,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960671603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:11,675 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/6973ae34b6d44faebde65df1e45ec969 2024-11-18T20:10:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0 2024-11-18T20:10:11,682 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0, entries=150, sequenceid=316, filesize=30.5 K 2024-11-18T20:10:11,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/5549fd203423457cbc4b4fbe63d22da1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/5549fd203423457cbc4b4fbe63d22da1 2024-11-18T20:10:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,687 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/5549fd203423457cbc4b4fbe63d22da1, entries=150, sequenceid=316, filesize=12.0 K 2024-11-18T20:10:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/6973ae34b6d44faebde65df1e45ec969 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6973ae34b6d44faebde65df1e45ec969 2024-11-18T20:10:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,695 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6973ae34b6d44faebde65df1e45ec969, entries=150, sequenceid=316, filesize=12.0 K 2024-11-18T20:10:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for c01d05a23f041191706ac16b56e11567 in 587ms, sequenceid=316, compaction requested=true 2024-11-18T20:10:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:11,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:11,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-18T20:10:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-18T20:10:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-18T20:10:11,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 741 msec 2024-11-18T20:10:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 754 msec 2024-11-18T20:10:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:11,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:11,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:10:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:11,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111876816a10d278444eab954d3973eb90b0_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:11,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742258_1434 (size=14994) 2024-11-18T20:10:12,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960672060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960672070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:10:12,092 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:12,093 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-18T20:10:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:12,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:12,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:12,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960672182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960672192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:12,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:12,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111876816a10d278444eab954d3973eb90b0_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111876816a10d278444eab954d3973eb90b0_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:12,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/7bd34c23bbb44b3a9ef4368500b4a3f9, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:12,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/7bd34c23bbb44b3a9ef4368500b4a3f9 is 175, key is test_row_0/A:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:12,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742259_1435 (size=39949) 2024-11-18T20:10:12,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960672402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960672403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:12,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41326 deadline: 1731960672535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,547 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_2' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41230 deadline: 1731960672542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,550 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_1' in c01d05a23f041191706ac16b56e11567 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:10:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:12,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:12,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960672712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960672725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:12,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/7bd34c23bbb44b3a9ef4368500b4a3f9 2024-11-18T20:10:12,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a3f0098cbbb54b48b5f5bb3ef26e1eae is 50, key is test_row_0/B:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:12,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742260_1436 (size=12301) 2024-11-18T20:10:12,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:12,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a3f0098cbbb54b48b5f5bb3ef26e1eae 2024-11-18T20:10:13,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/1315d45275c545529a46e5d5ee266d2f is 50, key is test_row_0/C:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:13,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742261_1437 (size=12301) 2024-11-18T20:10:13,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41300 deadline: 1731960673222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:13,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41244 deadline: 1731960673242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:13,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/1315d45275c545529a46e5d5ee266d2f 2024-11-18T20:10:13,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/7bd34c23bbb44b3a9ef4368500b4a3f9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9 2024-11-18T20:10:13,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9, entries=200, sequenceid=327, filesize=39.0 K 2024-11-18T20:10:13,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/a3f0098cbbb54b48b5f5bb3ef26e1eae as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a3f0098cbbb54b48b5f5bb3ef26e1eae 2024-11-18T20:10:13,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a3f0098cbbb54b48b5f5bb3ef26e1eae, entries=150, sequenceid=327, filesize=12.0 K 2024-11-18T20:10:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:13,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/1315d45275c545529a46e5d5ee266d2f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1315d45275c545529a46e5d5ee266d2f 2024-11-18T20:10:13,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1315d45275c545529a46e5d5ee266d2f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-18T20:10:13,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c01d05a23f041191706ac16b56e11567 in 1681ms, sequenceid=327, compaction requested=true 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:13,633 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c01d05a23f041191706ac16b56e11567:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:13,633 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:13,634 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:13,634 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:13,634 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/A is initiating minor compaction (all files) 2024-11-18T20:10:13,634 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/B is initiating minor compaction (all files) 2024-11-18T20:10:13,634 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/A in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,634 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/0dd3613b42754ec982edcd857efaa7e2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=131.2 K 2024-11-18T20:10:13,634 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,634 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/0dd3613b42754ec982edcd857efaa7e2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9] 2024-11-18T20:10:13,635 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd3613b42754ec982edcd857efaa7e2, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731960608135 2024-11-18T20:10:13,635 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b081fc376bb48b5b475f429a8ec996c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731960608788 2024-11-18T20:10:13,635 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/B in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,635 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a8b0fd3d1a48ed91bb1f68e6ceedf0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731960610064 2024-11-18T20:10:13,635 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/69488f98e04f4055ab0d51b272913f29, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/84d10013fa224a6f864f4e8de9a55722, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/5549fd203423457cbc4b4fbe63d22da1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a3f0098cbbb54b48b5f5bb3ef26e1eae] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=48.7 K 2024-11-18T20:10:13,635 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bd34c23bbb44b3a9ef4368500b4a3f9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731960611277 2024-11-18T20:10:13,635 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 69488f98e04f4055ab0d51b272913f29, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731960608135 2024-11-18T20:10:13,636 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d10013fa224a6f864f4e8de9a55722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731960608788 2024-11-18T20:10:13,636 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5549fd203423457cbc4b4fbe63d22da1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731960610064 2024-11-18T20:10:13,636 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f0098cbbb54b48b5f5bb3ef26e1eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731960611277 2024-11-18T20:10:13,641 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:13,649 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111883706067cd7145f2b3e745d1f3498122_c01d05a23f041191706ac16b56e11567 store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:13,651 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:13,651 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/acb9de3d16694f67a028b6d3aa12c5ac is 50, key is test_row_0/B:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:13,651 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111883706067cd7145f2b3e745d1f3498122_c01d05a23f041191706ac16b56e11567, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:13,651 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111883706067cd7145f2b3e745d1f3498122_c01d05a23f041191706ac16b56e11567 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:13,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742263_1439 (size=4469) 2024-11-18T20:10:13,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742262_1438 (size=13085) 2024-11-18T20:10:13,725 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-18T20:10:13,725 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:13,725 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,725 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:13,727 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-18T20:10:13,727 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:13,727 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,727 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:13,729 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-18T20:10:13,729 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:13,729 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,729 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:13,732 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-18T20:10:13,732 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:13,732 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,732 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:13,735 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-18T20:10:13,735 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:13,735 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,735 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:13,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:13,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T20:10:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:13,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:10:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111812fe0427bf4144a28fc10d851151874f_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960612067/Put/seqid=0 2024-11-18T20:10:13,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742264_1440 (size=12454) 2024-11-18T20:10:14,063 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#A#compaction#369 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:14,064 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/b620400d6ae241568678a85ddce21079 is 175, key is test_row_0/A:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742265_1441 (size=32039) 2024-11-18T20:10:14,070 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/acb9de3d16694f67a028b6d3aa12c5ac as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/acb9de3d16694f67a028b6d3aa12c5ac 2024-11-18T20:10:14,074 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c01d05a23f041191706ac16b56e11567/B of c01d05a23f041191706ac16b56e11567 into acb9de3d16694f67a028b6d3aa12c5ac(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:14,074 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:14,074 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/B, priority=12, startTime=1731960613633; duration=0sec 2024-11-18T20:10:14,074 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:14,074 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:B 2024-11-18T20:10:14,074 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:14,075 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:14,076 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): c01d05a23f041191706ac16b56e11567/C is initiating minor compaction (all files) 2024-11-18T20:10:14,076 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c01d05a23f041191706ac16b56e11567/C in TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:14,076 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8922b2dfca0748c596418db2a30b56e9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/93b495fea1fa4953acc6b7ecb44cbe6e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6973ae34b6d44faebde65df1e45ec969, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1315d45275c545529a46e5d5ee266d2f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp, totalSize=48.7 K 2024-11-18T20:10:14,076 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8922b2dfca0748c596418db2a30b56e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731960608135 2024-11-18T20:10:14,076 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b495fea1fa4953acc6b7ecb44cbe6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731960608788 2024-11-18T20:10:14,076 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6973ae34b6d44faebde65df1e45ec969, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731960610064 2024-11-18T20:10:14,077 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1315d45275c545529a46e5d5ee266d2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731960611277 2024-11-18T20:10:14,083 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c01d05a23f041191706ac16b56e11567#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:14,083 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/01ebe166f2fa47c39ffa906ee4cc51fe is 50, key is test_row_0/C:col10/1731960611949/Put/seqid=0 2024-11-18T20:10:14,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742266_1442 (size=13085) 2024-11-18T20:10:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:14,206 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111812fe0427bf4144a28fc10d851151874f_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111812fe0427bf4144a28fc10d851151874f_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/76c7f9f138a845d4acd126e3d1f7a3c1, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:14,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/76c7f9f138a845d4acd126e3d1f7a3c1 is 175, key is test_row_0/A:col10/1731960612067/Put/seqid=0 2024-11-18T20:10:14,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742267_1443 (size=31255) 2024-11-18T20:10:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:14,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. as already flushing 2024-11-18T20:10:14,233 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-18T20:10:14,233 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:14,233 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:14,233 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:14,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:14,263 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-18T20:10:14,263 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:14,263 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:14,263 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:14,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:14,475 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/b620400d6ae241568678a85ddce21079 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/b620400d6ae241568678a85ddce21079 2024-11-18T20:10:14,479 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c01d05a23f041191706ac16b56e11567/A of c01d05a23f041191706ac16b56e11567 into b620400d6ae241568678a85ddce21079(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:14,479 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:14,479 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/A, priority=12, startTime=1731960613633; duration=0sec 2024-11-18T20:10:14,479 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:14,479 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:A 2024-11-18T20:10:14,491 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/01ebe166f2fa47c39ffa906ee4cc51fe as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/01ebe166f2fa47c39ffa906ee4cc51fe 2024-11-18T20:10:14,495 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c01d05a23f041191706ac16b56e11567/C of c01d05a23f041191706ac16b56e11567 into 01ebe166f2fa47c39ffa906ee4cc51fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:14,495 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:14,495 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., storeName=c01d05a23f041191706ac16b56e11567/C, priority=12, startTime=1731960613633; duration=0sec 2024-11-18T20:10:14,495 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:14,495 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c01d05a23f041191706ac16b56e11567:C 2024-11-18T20:10:14,612 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/76c7f9f138a845d4acd126e3d1f7a3c1 2024-11-18T20:10:14,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/91c8b74854f74bfab84ef9d2f31d10c2 is 50, key is test_row_0/B:col10/1731960612067/Put/seqid=0 2024-11-18T20:10:14,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742268_1444 (size=12301) 2024-11-18T20:10:14,713 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-18T20:10:14,713 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:14,713 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:14,713 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:14,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:15,022 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/91c8b74854f74bfab84ef9d2f31d10c2 2024-11-18T20:10:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/ebf835d8264b4859a52f02d752e4d3de is 50, key is test_row_0/C:col10/1731960612067/Put/seqid=0 2024-11-18T20:10:15,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742269_1445 (size=12301) 2024-11-18T20:10:15,043 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/ebf835d8264b4859a52f02d752e4d3de 2024-11-18T20:10:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/76c7f9f138a845d4acd126e3d1f7a3c1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/76c7f9f138a845d4acd126e3d1f7a3c1 2024-11-18T20:10:15,052 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/76c7f9f138a845d4acd126e3d1f7a3c1, entries=150, sequenceid=352, filesize=30.5 K 2024-11-18T20:10:15,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/91c8b74854f74bfab84ef9d2f31d10c2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/91c8b74854f74bfab84ef9d2f31d10c2 2024-11-18T20:10:15,057 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/91c8b74854f74bfab84ef9d2f31d10c2, entries=150, sequenceid=352, filesize=12.0 K 2024-11-18T20:10:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/ebf835d8264b4859a52f02d752e4d3de as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/ebf835d8264b4859a52f02d752e4d3de 2024-11-18T20:10:15,061 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/ebf835d8264b4859a52f02d752e4d3de, entries=150, sequenceid=352, filesize=12.0 K 2024-11-18T20:10:15,062 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=20.13 KB/20610 for c01d05a23f041191706ac16b56e11567 in 1280ms, sequenceid=352, compaction requested=false 2024-11-18T20:10:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for c01d05a23f041191706ac16b56e11567: 2024-11-18T20:10:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-18T20:10:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-18T20:10:15,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-18T20:10:15,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9690 sec 2024-11-18T20:10:15,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.9730 sec 2024-11-18T20:10:15,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:10:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:10:16,262 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:19,790 WARN [regionserver/d8f95bb18b58:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-18T20:10:19,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ec5a86d8bbdc47359fc41c2cd90c37b5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7ca51dc108e34e7482e031d6f2d11d04, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/42b3cfafe8c2497b8a5bdd42c6a5d80b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/299eb34f51ce40bfb05571dea9ae1df2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae65911c5db64665aa22afefb817b446, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/0dd3613b42754ec982edcd857efaa7e2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9] to archive 2024-11-18T20:10:19,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:10:19,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/933b93a49e0a46639c76954ce7cc5a95 2024-11-18T20:10:19,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5a43d9f9c77743938059459dfacf81a2 2024-11-18T20:10:19,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/718d15048ffe47919bf59e0fc55f1833 2024-11-18T20:10:19,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ec5a86d8bbdc47359fc41c2cd90c37b5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ec5a86d8bbdc47359fc41c2cd90c37b5 2024-11-18T20:10:19,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/1243b4cb74a643cbababa1f2cda17b09 2024-11-18T20:10:19,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/87c5cace22564e0f9c7dc0b8119e7439 2024-11-18T20:10:19,804 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7ca51dc108e34e7482e031d6f2d11d04 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7ca51dc108e34e7482e031d6f2d11d04 2024-11-18T20:10:19,805 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/2080bcea30574ca787333bd1f6d4f7ac 2024-11-18T20:10:19,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/8c216943b69b4893b7aa3b0df1bd47d3 2024-11-18T20:10:19,808 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/20a400ff64634b1cba5e0dfcfeea95fe 2024-11-18T20:10:19,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/42b3cfafe8c2497b8a5bdd42c6a5d80b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/42b3cfafe8c2497b8a5bdd42c6a5d80b 2024-11-18T20:10:19,810 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/5f35717c73ce434db652fc5b61397d0a 2024-11-18T20:10:19,812 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/d5122d7df7624ddd8bb73c46631a0255 2024-11-18T20:10:19,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/299eb34f51ce40bfb05571dea9ae1df2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/299eb34f51ce40bfb05571dea9ae1df2 2024-11-18T20:10:19,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae8837829f984dd1ab668c6d49957d46 2024-11-18T20:10:19,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a2a4732a6ab847528466b4b0078e2214 2024-11-18T20:10:19,817 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae65911c5db64665aa22afefb817b446 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/ae65911c5db64665aa22afefb817b446 2024-11-18T20:10:19,818 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/90a08a6bc23b4221bc9fb926d1780fd0 2024-11-18T20:10:19,820 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/0dd3613b42754ec982edcd857efaa7e2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/0dd3613b42754ec982edcd857efaa7e2 2024-11-18T20:10:19,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f782be5bfc4e4e309f927c8d84455f20 2024-11-18T20:10:19,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/6b081fc376bb48b5b475f429a8ec996c 2024-11-18T20:10:19,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/a4a8b0fd3d1a48ed91bb1f68e6ceedf0 2024-11-18T20:10:19,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/7bd34c23bbb44b3a9ef4368500b4a3f9 2024-11-18T20:10:19,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a0cb1a66f0744effbbd76d303ef397e6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/6dd1d55867584f6fb84be829a5855286, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a85f6a3aa36546a2804eadfc9414ddef, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0c60acaf0c8f4aa8885449888f825be6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/7540c2a7210e4024a9448dbcd0d23710, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0cc6c006a94145cf8f15fd85a9267a8f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/fe63a80bf31341ddb08d0682dadb34a8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/06cbebce9b3a4e3db5286d903a63bcf9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/51a9b1e4e505402ca674f8e18835e26c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3ec9658df6064da198976b6e6b7abf99, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1ec66b377d0d4beebe7be92969cc641f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/f0914e9987f94c56a399c975ea4da34f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1240aa908cd349c389d97024385c134b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/eb36bb76f65848c5abee7662b7bb7867, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/15688e4cdb54489fbfbe5b2e346ea0bf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a2365486ddfd40648d3068dc8a3fd897, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e90984767134285b4cb2c7188c091cd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e8e065d8ee74229927f862ad5b00ac9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/69488f98e04f4055ab0d51b272913f29, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/822c54c7d1644e859d24eb26688ff595, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/84d10013fa224a6f864f4e8de9a55722, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/5549fd203423457cbc4b4fbe63d22da1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a3f0098cbbb54b48b5f5bb3ef26e1eae] to archive 2024-11-18T20:10:19,831 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:10:19,832 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a0cb1a66f0744effbbd76d303ef397e6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a0cb1a66f0744effbbd76d303ef397e6 2024-11-18T20:10:19,834 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/6dd1d55867584f6fb84be829a5855286 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/6dd1d55867584f6fb84be829a5855286 2024-11-18T20:10:19,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a85f6a3aa36546a2804eadfc9414ddef to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a85f6a3aa36546a2804eadfc9414ddef 2024-11-18T20:10:19,837 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0c60acaf0c8f4aa8885449888f825be6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0c60acaf0c8f4aa8885449888f825be6 2024-11-18T20:10:19,838 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/7540c2a7210e4024a9448dbcd0d23710 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/7540c2a7210e4024a9448dbcd0d23710 2024-11-18T20:10:19,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0cc6c006a94145cf8f15fd85a9267a8f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/0cc6c006a94145cf8f15fd85a9267a8f 2024-11-18T20:10:19,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/fe63a80bf31341ddb08d0682dadb34a8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/fe63a80bf31341ddb08d0682dadb34a8 2024-11-18T20:10:19,842 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/06cbebce9b3a4e3db5286d903a63bcf9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/06cbebce9b3a4e3db5286d903a63bcf9 2024-11-18T20:10:19,843 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/51a9b1e4e505402ca674f8e18835e26c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/51a9b1e4e505402ca674f8e18835e26c 2024-11-18T20:10:19,845 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3ec9658df6064da198976b6e6b7abf99 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3ec9658df6064da198976b6e6b7abf99 2024-11-18T20:10:19,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1ec66b377d0d4beebe7be92969cc641f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1ec66b377d0d4beebe7be92969cc641f 2024-11-18T20:10:19,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/f0914e9987f94c56a399c975ea4da34f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/f0914e9987f94c56a399c975ea4da34f 2024-11-18T20:10:19,850 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1240aa908cd349c389d97024385c134b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/1240aa908cd349c389d97024385c134b 2024-11-18T20:10:19,851 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/eb36bb76f65848c5abee7662b7bb7867 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/eb36bb76f65848c5abee7662b7bb7867 2024-11-18T20:10:19,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/15688e4cdb54489fbfbe5b2e346ea0bf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/15688e4cdb54489fbfbe5b2e346ea0bf 2024-11-18T20:10:19,854 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a2365486ddfd40648d3068dc8a3fd897 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a2365486ddfd40648d3068dc8a3fd897 2024-11-18T20:10:19,855 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e90984767134285b4cb2c7188c091cd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e90984767134285b4cb2c7188c091cd 2024-11-18T20:10:19,856 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e8e065d8ee74229927f862ad5b00ac9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/3e8e065d8ee74229927f862ad5b00ac9 2024-11-18T20:10:19,857 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/69488f98e04f4055ab0d51b272913f29 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/69488f98e04f4055ab0d51b272913f29 2024-11-18T20:10:19,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/822c54c7d1644e859d24eb26688ff595 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/822c54c7d1644e859d24eb26688ff595 2024-11-18T20:10:19,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/84d10013fa224a6f864f4e8de9a55722 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/84d10013fa224a6f864f4e8de9a55722 2024-11-18T20:10:19,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/5549fd203423457cbc4b4fbe63d22da1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/5549fd203423457cbc4b4fbe63d22da1 2024-11-18T20:10:19,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a3f0098cbbb54b48b5f5bb3ef26e1eae to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/a3f0098cbbb54b48b5f5bb3ef26e1eae 2024-11-18T20:10:19,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/bc610d1e356e4bd29bf0d77cf4349f21, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8289c9ba10c44e0597fa3527ef5db54c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1c71af87055f44f9a559317d2907e9e3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/5061524600a14981ace2c59cc5a71e71, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d626aede5dd49179e6678f5cb4b5444, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/0ff46429feac4a27a05bfa4a332d0064, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/f49750348e9e4144b25e1bd908272b33, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/708e6ba2a94b4525a8f9974e22d2b8bb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/748fd90386004dd3bf287dab860e27c5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/404adefbe4684ab5927ab74fb147338a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6e33984051cc46afa4bf448e1e88eb83, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d73cd9cbd3148ec94dfbc27c3fc35f4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/c3fea0084bad4c5481984754dc4388fb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b288c42f3f234543ad1c7f9e97180718, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b847ff9df2084e15876f0fcda97a4254, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/316aae6f6bec4e169e4fde4fb6682f64, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b6644d0c20c04b4d87f806f98ef162ad, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8922b2dfca0748c596418db2a30b56e9, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/341e1e3c4adf4b7d9d7f9ed71208edcb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/93b495fea1fa4953acc6b7ecb44cbe6e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6973ae34b6d44faebde65df1e45ec969, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1315d45275c545529a46e5d5ee266d2f] to archive 2024-11-18T20:10:19,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:10:19,871 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/bc610d1e356e4bd29bf0d77cf4349f21 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/bc610d1e356e4bd29bf0d77cf4349f21 2024-11-18T20:10:19,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8289c9ba10c44e0597fa3527ef5db54c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8289c9ba10c44e0597fa3527ef5db54c 2024-11-18T20:10:19,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1c71af87055f44f9a559317d2907e9e3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1c71af87055f44f9a559317d2907e9e3 2024-11-18T20:10:19,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/5061524600a14981ace2c59cc5a71e71 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/5061524600a14981ace2c59cc5a71e71 2024-11-18T20:10:19,876 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d626aede5dd49179e6678f5cb4b5444 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d626aede5dd49179e6678f5cb4b5444 2024-11-18T20:10:19,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/0ff46429feac4a27a05bfa4a332d0064 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/0ff46429feac4a27a05bfa4a332d0064 2024-11-18T20:10:19,878 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/f49750348e9e4144b25e1bd908272b33 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/f49750348e9e4144b25e1bd908272b33 2024-11-18T20:10:19,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/708e6ba2a94b4525a8f9974e22d2b8bb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/708e6ba2a94b4525a8f9974e22d2b8bb 2024-11-18T20:10:19,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/748fd90386004dd3bf287dab860e27c5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/748fd90386004dd3bf287dab860e27c5 2024-11-18T20:10:19,881 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/404adefbe4684ab5927ab74fb147338a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/404adefbe4684ab5927ab74fb147338a 2024-11-18T20:10:19,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6e33984051cc46afa4bf448e1e88eb83 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6e33984051cc46afa4bf448e1e88eb83 2024-11-18T20:10:19,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d73cd9cbd3148ec94dfbc27c3fc35f4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/4d73cd9cbd3148ec94dfbc27c3fc35f4 2024-11-18T20:10:19,885 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/c3fea0084bad4c5481984754dc4388fb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/c3fea0084bad4c5481984754dc4388fb 2024-11-18T20:10:19,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b288c42f3f234543ad1c7f9e97180718 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b288c42f3f234543ad1c7f9e97180718 2024-11-18T20:10:19,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b09fa42d7e7b4ef2ab6c7ac121d3dc6f 2024-11-18T20:10:19,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b847ff9df2084e15876f0fcda97a4254 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b847ff9df2084e15876f0fcda97a4254 2024-11-18T20:10:19,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/316aae6f6bec4e169e4fde4fb6682f64 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/316aae6f6bec4e169e4fde4fb6682f64 2024-11-18T20:10:19,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b6644d0c20c04b4d87f806f98ef162ad to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/b6644d0c20c04b4d87f806f98ef162ad 2024-11-18T20:10:19,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8922b2dfca0748c596418db2a30b56e9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/8922b2dfca0748c596418db2a30b56e9 2024-11-18T20:10:19,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/341e1e3c4adf4b7d9d7f9ed71208edcb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/341e1e3c4adf4b7d9d7f9ed71208edcb 2024-11-18T20:10:19,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/93b495fea1fa4953acc6b7ecb44cbe6e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/93b495fea1fa4953acc6b7ecb44cbe6e 2024-11-18T20:10:19,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6973ae34b6d44faebde65df1e45ec969 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/6973ae34b6d44faebde65df1e45ec969 2024-11-18T20:10:19,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d8f95bb18b58:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1315d45275c545529a46e5d5ee266d2f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/1315d45275c545529a46e5d5ee266d2f 2024-11-18T20:10:22,604 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-18T20:10:22,604 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:22,604 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:22,604 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:22,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:22,623 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-18T20:10:22,623 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:22,624 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:22,624 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:22,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3674 rows 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1227 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3680 rows 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1228 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3679 rows 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3651 rows 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1232 2024-11-18T20:10:22,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3690 rows 2024-11-18T20:10:22,624 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:10:22,625 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:10:22,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:22,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:22,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:22,626 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:10:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:10:22,631 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960622630"}]},"ts":"1731960622630"} 2024-11-18T20:10:22,632 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:10:22,632 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:10:22,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:10:22,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, UNASSIGN}] 2024-11-18T20:10:22,636 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, UNASSIGN 2024-11-18T20:10:22,637 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:22,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, UNASSIGN because future has completed 2024-11-18T20:10:22,639 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:10:22,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:10:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:10:22,792 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:22,792 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:10:22,792 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing c01d05a23f041191706ac16b56e11567, disabling compactions & flushes 2024-11-18T20:10:22,792 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:22,792 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:22,793 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. after waiting 0 ms 2024-11-18T20:10:22,793 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:22,793 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing c01d05a23f041191706ac16b56e11567 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:10:22,799 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186e868726b4134b5dbb6a7c021c0adce2_c01d05a23f041191706ac16b56e11567 is 50, key is test_row_0/A:col10/1731960614712/Put/seqid=0 2024-11-18T20:10:22,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742270_1446 (size=12454) 2024-11-18T20:10:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:10:23,224 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:23,232 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186e868726b4134b5dbb6a7c021c0adce2_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186e868726b4134b5dbb6a7c021c0adce2_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:23,233 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f242b95645534962841b56640561da6e, store: [table=TestAcidGuarantees family=A region=c01d05a23f041191706ac16b56e11567] 2024-11-18T20:10:23,233 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f242b95645534962841b56640561da6e is 175, key is test_row_0/A:col10/1731960614712/Put/seqid=0 2024-11-18T20:10:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:10:23,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742271_1447 (size=31255) 2024-11-18T20:10:23,653 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f242b95645534962841b56640561da6e 2024-11-18T20:10:23,661 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/e6fdae6ad5ee4c6eb63b275c1b877c31 is 50, key is test_row_0/B:col10/1731960614712/Put/seqid=0 2024-11-18T20:10:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742272_1448 (size=12301) 2024-11-18T20:10:23,685 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/e6fdae6ad5ee4c6eb63b275c1b877c31 2024-11-18T20:10:23,695 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/75c672243ec14f159e52bd9fe657bd90 is 50, key is test_row_0/C:col10/1731960614712/Put/seqid=0 2024-11-18T20:10:23,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742273_1449 (size=12301) 2024-11-18T20:10:23,741 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/75c672243ec14f159e52bd9fe657bd90 2024-11-18T20:10:23,756 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/A/f242b95645534962841b56640561da6e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f242b95645534962841b56640561da6e 2024-11-18T20:10:23,761 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f242b95645534962841b56640561da6e, entries=150, sequenceid=363, filesize=30.5 K 2024-11-18T20:10:23,762 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/B/e6fdae6ad5ee4c6eb63b275c1b877c31 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/e6fdae6ad5ee4c6eb63b275c1b877c31 2024-11-18T20:10:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:10:23,766 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/e6fdae6ad5ee4c6eb63b275c1b877c31, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:10:23,767 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/.tmp/C/75c672243ec14f159e52bd9fe657bd90 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/75c672243ec14f159e52bd9fe657bd90 2024-11-18T20:10:23,771 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/75c672243ec14f159e52bd9fe657bd90, entries=150, sequenceid=363, filesize=12.0 K 2024-11-18T20:10:23,772 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c01d05a23f041191706ac16b56e11567 in 979ms, sequenceid=363, compaction requested=true 2024-11-18T20:10:23,815 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-18T20:10:23,817 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. 2024-11-18T20:10:23,817 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for c01d05a23f041191706ac16b56e11567: Waiting for close lock at 1731960622792Running coprocessor pre-close hooks at 1731960622792Disabling compacts and flushes for region at 1731960622792Disabling writes for close at 1731960622793 (+1 ms)Obtaining lock to block concurrent updates at 1731960622793Preparing flush snapshotting stores in c01d05a23f041191706ac16b56e11567 at 1731960622793Finished memstore snapshotting TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960622793Flushing stores of TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567. at 1731960622794 (+1 ms)Flushing c01d05a23f041191706ac16b56e11567/A: creating writer at 1731960622794Flushing mob file c01d05a23f041191706ac16b56e11567/A: appending metadata at 1731960622799 (+5 ms)Flushing mob file c01d05a23f041191706ac16b56e11567/A: closing flushed file at 1731960622799Flushing c01d05a23f041191706ac16b56e11567/A: appending metadata at 1731960623233 (+434 ms)Flushing c01d05a23f041191706ac16b56e11567/A: closing flushed file at 1731960623233Flushing c01d05a23f041191706ac16b56e11567/B: creating writer at 1731960623657 (+424 ms)Flushing c01d05a23f041191706ac16b56e11567/B: appending metadata at 1731960623660 (+3 ms)Flushing c01d05a23f041191706ac16b56e11567/B: closing flushed file at 1731960623660Flushing c01d05a23f041191706ac16b56e11567/C: creating writer at 1731960623690 (+30 ms)Flushing c01d05a23f041191706ac16b56e11567/C: appending metadata at 1731960623694 (+4 ms)Flushing c01d05a23f041191706ac16b56e11567/C: closing flushed file at 1731960623694Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3802a5e4: reopening flushed file at 1731960623755 (+61 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25df2cb6: reopening flushed file at 1731960623761 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fa5dfa2: reopening flushed file at 1731960623766 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c01d05a23f041191706ac16b56e11567 in 979ms, sequenceid=363, compaction requested=true at 1731960623772 (+6 ms)Writing region close event to WAL at 1731960623805 (+33 ms)Running coprocessor post-close hooks at 1731960623817 (+12 ms)Closed at 1731960623817 2024-11-18T20:10:23,833 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:23,839 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c01d05a23f041191706ac16b56e11567, regionState=CLOSED 2024-11-18T20:10:23,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:10:23,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-18T20:10:23,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure c01d05a23f041191706ac16b56e11567, server=d8f95bb18b58,42447,1731960497538 in 1.2240 sec 2024-11-18T20:10:23,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-18T20:10:23,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c01d05a23f041191706ac16b56e11567, UNASSIGN in 1.2570 sec 2024-11-18T20:10:23,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-18T20:10:23,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2820 sec 2024-11-18T20:10:23,918 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960623918"}]},"ts":"1731960623918"} 2024-11-18T20:10:23,920 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:10:23,920 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:10:23,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2950 sec 2024-11-18T20:10:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T20:10:24,773 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:24,773 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:10:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:24,776 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:10:24,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:24,779 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,781 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/recovered.edits] 2024-11-18T20:10:24,784 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/76c7f9f138a845d4acd126e3d1f7a3c1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/76c7f9f138a845d4acd126e3d1f7a3c1 2024-11-18T20:10:24,785 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/b620400d6ae241568678a85ddce21079 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/b620400d6ae241568678a85ddce21079 2024-11-18T20:10:24,786 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f242b95645534962841b56640561da6e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/A/f242b95645534962841b56640561da6e 2024-11-18T20:10:24,789 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/91c8b74854f74bfab84ef9d2f31d10c2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/91c8b74854f74bfab84ef9d2f31d10c2 2024-11-18T20:10:24,790 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/acb9de3d16694f67a028b6d3aa12c5ac to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/acb9de3d16694f67a028b6d3aa12c5ac 2024-11-18T20:10:24,791 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/e6fdae6ad5ee4c6eb63b275c1b877c31 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/B/e6fdae6ad5ee4c6eb63b275c1b877c31 2024-11-18T20:10:24,803 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/01ebe166f2fa47c39ffa906ee4cc51fe to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/01ebe166f2fa47c39ffa906ee4cc51fe 2024-11-18T20:10:24,804 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/75c672243ec14f159e52bd9fe657bd90 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/75c672243ec14f159e52bd9fe657bd90 2024-11-18T20:10:24,806 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/ebf835d8264b4859a52f02d752e4d3de to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/C/ebf835d8264b4859a52f02d752e4d3de 2024-11-18T20:10:24,809 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/recovered.edits/366.seqid to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567/recovered.edits/366.seqid 2024-11-18T20:10:24,809 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,810 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:10:24,810 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:10:24,811 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:10:24,815 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180868fc2a83864cfcb743dc03da75cb6c_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180868fc2a83864cfcb743dc03da75cb6c_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,816 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180f8eb6b39ed8479eb292f023b28b93bd_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180f8eb6b39ed8479eb292f023b28b93bd_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,818 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111812fe0427bf4144a28fc10d851151874f_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111812fe0427bf4144a28fc10d851151874f_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,819 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111817d3e8d6a3414ceba4d5c4fa75b74b56_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111817d3e8d6a3414ceba4d5c4fa75b74b56_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,820 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181f6de5f8ff18482e93b3448112e64b98_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181f6de5f8ff18482e93b3448112e64b98_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,822 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182cd98e1966d241d8b953b6da7bdeb165_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182cd98e1966d241d8b953b6da7bdeb165_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,823 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111831b52f37e84a4b918573267b08f987db_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111831b52f37e84a4b918573267b08f987db_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,825 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184d24440b196a474b8c7d99ac4584638d_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184d24440b196a474b8c7d99ac4584638d_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,826 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186e868726b4134b5dbb6a7c021c0adce2_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186e868726b4134b5dbb6a7c021c0adce2_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,828 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187600ff26e21541379b9b7ba57b46af46_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187600ff26e21541379b9b7ba57b46af46_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,829 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111876816a10d278444eab954d3973eb90b0_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111876816a10d278444eab954d3973eb90b0_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,830 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188338fa10cdba4913bd9e89c0c7a020ef_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188338fa10cdba4913bd9e89c0c7a020ef_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,832 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188be103c461fa414bacf4f5e069a04e02_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188be103c461fa414bacf4f5e069a04e02_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,833 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a7ffa4855de14b02864450f0b07cee52_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a7ffa4855de14b02864450f0b07cee52_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,834 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ba1cc79061c1461586faca403f32adbe_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ba1cc79061c1461586faca403f32adbe_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,836 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5edd615a04c4070938e0bfb7ad5be3a_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d5edd615a04c4070938e0bfb7ad5be3a_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,837 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dc4e28706451435889977fb23cc3d646_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dc4e28706451435889977fb23cc3d646_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,840 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dd765c445296459db2e3bf117d8d5a5c_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118dd765c445296459db2e3bf117d8d5a5c_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,841 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f0a3963bd2f04b50b73edcafd88ff826_c01d05a23f041191706ac16b56e11567 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f0a3963bd2f04b50b73edcafd88ff826_c01d05a23f041191706ac16b56e11567 2024-11-18T20:10:24,842 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:10:24,845 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:24,848 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:10:24,850 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:10:24,852 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:24,852 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:10:24,852 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960624852"}]},"ts":"9223372036854775807"} 2024-11-18T20:10:24,856 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:10:24,856 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c01d05a23f041191706ac16b56e11567, NAME => 'TestAcidGuarantees,,1731960591581.c01d05a23f041191706ac16b56e11567.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:10:24,856 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:10:24,856 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960624856"}]},"ts":"9223372036854775807"} 2024-11-18T20:10:24,877 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:10:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:10:24,885 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:24,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 112 msec 2024-11-18T20:10:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T20:10:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:10:25,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:25,104 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 240), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=587 (was 484) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1400 (was 1962) 2024-11-18T20:10:25,115 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=587, ProcessCount=11, AvailableMemoryMB=1399 2024-11-18T20:10:25,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:10:25,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:10:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:25,120 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:10:25,120 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:25,120 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-18T20:10:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-18T20:10:25,121 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:10:25,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742274_1450 (size=959) 2024-11-18T20:10:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-18T20:10:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-18T20:10:25,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:10:25,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742275_1451 (size=53) 2024-11-18T20:10:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-18T20:10:25,997 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:10:25,997 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 471b1d433f657c986c9e035c4a593a36, disabling compactions & flushes 2024-11-18T20:10:25,997 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:25,997 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:25,997 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. after waiting 0 ms 2024-11-18T20:10:25,997 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:25,997 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:25,997 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 471b1d433f657c986c9e035c4a593a36: Waiting for close lock at 1731960625997Disabling compacts and flushes for region at 1731960625997Disabling writes for close at 1731960625997Writing region close event to WAL at 1731960625997Closed at 1731960625997 2024-11-18T20:10:25,999 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:10:25,999 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960625999"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960625999"}]},"ts":"1731960625999"} 2024-11-18T20:10:26,001 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:10:26,012 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:10:26,012 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960626012"}]},"ts":"1731960626012"} 2024-11-18T20:10:26,013 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:10:26,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, ASSIGN}] 2024-11-18T20:10:26,015 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, ASSIGN 2024-11-18T20:10:26,016 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, ASSIGN; state=OFFLINE, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=false 2024-11-18T20:10:26,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=471b1d433f657c986c9e035c4a593a36, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:26,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, ASSIGN because future has completed 2024-11-18T20:10:26,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:10:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-18T20:10:26,323 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:26,323 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:10:26,324 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,324 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:10:26,324 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,324 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,326 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,327 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 471b1d433f657c986c9e035c4a593a36 columnFamilyName A 2024-11-18T20:10:26,327 DEBUG [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:26,328 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] regionserver.HStore(327): Store=471b1d433f657c986c9e035c4a593a36/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:26,328 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,329 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 471b1d433f657c986c9e035c4a593a36 columnFamilyName B 2024-11-18T20:10:26,329 DEBUG [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:26,329 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] regionserver.HStore(327): Store=471b1d433f657c986c9e035c4a593a36/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:26,330 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,331 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 471b1d433f657c986c9e035c4a593a36 columnFamilyName C 2024-11-18T20:10:26,331 DEBUG [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:26,331 INFO [StoreOpener-471b1d433f657c986c9e035c4a593a36-1 {}] regionserver.HStore(327): Store=471b1d433f657c986c9e035c4a593a36/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:26,332 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,332 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,333 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,334 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,334 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,334 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:10:26,335 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,337 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:10:26,338 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 471b1d433f657c986c9e035c4a593a36; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70031313, jitterRate=0.04354788362979889}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:10:26,338 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:26,338 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 471b1d433f657c986c9e035c4a593a36: Running coprocessor pre-open hook at 1731960626325Writing region info on filesystem at 1731960626325Initializing all the Stores at 1731960626325Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960626325Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960626326 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960626326Cleaning up temporary data from old regions at 1731960626334 (+8 ms)Running coprocessor post-open hooks at 1731960626338 (+4 ms)Region opened successfully at 1731960626338 2024-11-18T20:10:26,341 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., pid=122, masterSystemTime=1731960626320 2024-11-18T20:10:26,342 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:26,343 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:26,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=471b1d433f657c986c9e035c4a593a36, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:26,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:10:26,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-18T20:10:26,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 in 178 msec 2024-11-18T20:10:26,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-18T20:10:26,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, ASSIGN in 335 msec 2024-11-18T20:10:26,351 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:10:26,351 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960626351"}]},"ts":"1731960626351"} 2024-11-18T20:10:26,352 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:10:26,353 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:10:26,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2370 sec 2024-11-18T20:10:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-18T20:10:27,272 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:27,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,274 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf38639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,276 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f2d811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,281 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:10:27,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:10:27,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@15981fc7 2024-11-18T20:10:27,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:10:27,284 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:10:27,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,290 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54670e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6d5a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,303 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d9ccf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,308 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250a263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,313 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1072b262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6777ecd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,335 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b92a620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b5e322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d45bbc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,350 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9d42ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17eed8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,360 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a24437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58bfdd7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,373 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d09f163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,378 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1508ae8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,380 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5c4a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab0b925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,386 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d99dedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:27,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:27,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:27,393 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb0893d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:27,395 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,395 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,396 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,400 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:27,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-18T20:10:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,414 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:27,419 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,419 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:27,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:27,420 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,420 DEBUG [Thread-2025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,422 DEBUG [Thread-2027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,424 DEBUG [Thread-2023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,425 DEBUG [Thread-2023 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,425 DEBUG [Thread-2023 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,435 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:27,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2] 2024-11-18T20:10:27,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:27,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:10:27,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/4a00b8a71d4b49cd9a4d3d847e68d111 is 50, key is test_row_0/A:col10/1731960627450/Put/seqid=0 2024-11-18T20:10:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:27,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:27,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742276_1452 (size=12001) 2024-11-18T20:10:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960687548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960687549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960687568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960687576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960687577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960687683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960687692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960687693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960687696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960687702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:27,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:27,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:27,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960687902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960687912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960687912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960687915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960687922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:27,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:27,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:27,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/4a00b8a71d4b49cd9a4d3d847e68d111 2024-11-18T20:10:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:28,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:28,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/a2b133365c524d869f0bb48903d61d56 is 50, key is test_row_0/B:col10/1731960627450/Put/seqid=0 2024-11-18T20:10:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742277_1453 (size=12001) 2024-11-18T20:10:28,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960688223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960688223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960688224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960688232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960688234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/a2b133365c524d869f0bb48903d61d56 2024-11-18T20:10:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:28,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:28,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/33c23f393cd04ffb9f0ffe189511af89 is 50, key is test_row_0/C:col10/1731960627450/Put/seqid=0 2024-11-18T20:10:28,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742278_1454 (size=12001) 2024-11-18T20:10:28,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/33c23f393cd04ffb9f0ffe189511af89 2024-11-18T20:10:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/4a00b8a71d4b49cd9a4d3d847e68d111 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/4a00b8a71d4b49cd9a4d3d847e68d111 2024-11-18T20:10:28,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/4a00b8a71d4b49cd9a4d3d847e68d111, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:10:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/a2b133365c524d869f0bb48903d61d56 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/a2b133365c524d869f0bb48903d61d56 2024-11-18T20:10:28,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/a2b133365c524d869f0bb48903d61d56, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:10:28,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/33c23f393cd04ffb9f0ffe189511af89 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/33c23f393cd04ffb9f0ffe189511af89 2024-11-18T20:10:28,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/33c23f393cd04ffb9f0ffe189511af89, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:10:28,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 471b1d433f657c986c9e035c4a593a36 in 1275ms, sequenceid=15, compaction requested=false 2024-11-18T20:10:28,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:28,754 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:28,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:10:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:28,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/44171d59fedf497f9d20331b964ab625 is 50, key is test_row_0/A:col10/1731960627560/Put/seqid=0 2024-11-18T20:10:28,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742279_1455 (size=14341) 2024-11-18T20:10:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960688788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960688809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960688810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960688816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960688823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:28,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:28,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:28,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960688924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960688936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960688942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960688942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960688943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960689136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960689156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960689162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960689162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960689163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/44171d59fedf497f9d20331b964ab625 2024-11-18T20:10:29,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d431c6e580cb4980adf6b73578e8da18 is 50, key is test_row_0/B:col10/1731960627560/Put/seqid=0 2024-11-18T20:10:29,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742280_1456 (size=12001) 2024-11-18T20:10:29,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d431c6e580cb4980adf6b73578e8da18 2024-11-18T20:10:29,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/8b037ee2e23b4d168af7727b7d6d80a2 is 50, key is test_row_0/C:col10/1731960627560/Put/seqid=0 2024-11-18T20:10:29,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,368 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:10:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742281_1457 (size=12001) 2024-11-18T20:10:29,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960689454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960689462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960689472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960689482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960689483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:29,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:29,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/8b037ee2e23b4d168af7727b7d6d80a2 2024-11-18T20:10:29,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/44171d59fedf497f9d20331b964ab625 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/44171d59fedf497f9d20331b964ab625 2024-11-18T20:10:29,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/44171d59fedf497f9d20331b964ab625, entries=200, sequenceid=39, filesize=14.0 K 2024-11-18T20:10:29,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d431c6e580cb4980adf6b73578e8da18 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d431c6e580cb4980adf6b73578e8da18 2024-11-18T20:10:29,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d431c6e580cb4980adf6b73578e8da18, entries=150, sequenceid=39, filesize=11.7 K 2024-11-18T20:10:29,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/8b037ee2e23b4d168af7727b7d6d80a2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/8b037ee2e23b4d168af7727b7d6d80a2 2024-11-18T20:10:29,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/8b037ee2e23b4d168af7727b7d6d80a2, entries=150, sequenceid=39, filesize=11.7 K 2024-11-18T20:10:29,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 471b1d433f657c986c9e035c4a593a36 in 1170ms, sequenceid=39, compaction requested=false 2024-11-18T20:10:29,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:29,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:29,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:10:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:29,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:29,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/ec272de7c6ee4446be4998a9dcc7e5b2 is 50, key is test_row_0/A:col10/1731960629988/Put/seqid=0 2024-11-18T20:10:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742282_1458 (size=14341) 2024-11-18T20:10:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960690165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960690172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960690174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960690176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960690185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960690295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960690303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960690303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960690305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960690313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:30,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/ec272de7c6ee4446be4998a9dcc7e5b2 2024-11-18T20:10:30,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:30,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:30,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/c89b26afa2b346b89480f555100af068 is 50, key is test_row_0/B:col10/1731960629988/Put/seqid=0 2024-11-18T20:10:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960690516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960690524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960690526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960690526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960690527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742283_1459 (size=12001) 2024-11-18T20:10:30,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/c89b26afa2b346b89480f555100af068 2024-11-18T20:10:30,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:30,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:30,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/db586b7f9cf74423a2e7394035519a9f is 50, key is test_row_0/C:col10/1731960629988/Put/seqid=0 2024-11-18T20:10:30,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742284_1460 (size=12001) 2024-11-18T20:10:30,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960690835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960690843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960690843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960690845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960690859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:30,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:30,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:31,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/db586b7f9cf74423a2e7394035519a9f 2024-11-18T20:10:31,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/ec272de7c6ee4446be4998a9dcc7e5b2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ec272de7c6ee4446be4998a9dcc7e5b2 2024-11-18T20:10:31,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ec272de7c6ee4446be4998a9dcc7e5b2, entries=200, sequenceid=52, filesize=14.0 K 2024-11-18T20:10:31,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:31,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/c89b26afa2b346b89480f555100af068 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/c89b26afa2b346b89480f555100af068 2024-11-18T20:10:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:31,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/c89b26afa2b346b89480f555100af068, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T20:10:31,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/db586b7f9cf74423a2e7394035519a9f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/db586b7f9cf74423a2e7394035519a9f 2024-11-18T20:10:31,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/db586b7f9cf74423a2e7394035519a9f, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T20:10:31,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 471b1d433f657c986c9e035c4a593a36 in 1158ms, sequenceid=52, compaction requested=true 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:31,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T20:10:31,148 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:31,148 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:31,158 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:31,158 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:31,158 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,158 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/4a00b8a71d4b49cd9a4d3d847e68d111, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/44171d59fedf497f9d20331b964ab625, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ec272de7c6ee4446be4998a9dcc7e5b2] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=39.7 K 2024-11-18T20:10:31,159 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:31,159 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:31,159 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,159 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/33c23f393cd04ffb9f0ffe189511af89, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/8b037ee2e23b4d168af7727b7d6d80a2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/db586b7f9cf74423a2e7394035519a9f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=35.2 K 2024-11-18T20:10:31,159 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a00b8a71d4b49cd9a4d3d847e68d111, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960627435 2024-11-18T20:10:31,159 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c23f393cd04ffb9f0ffe189511af89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960627435 2024-11-18T20:10:31,160 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44171d59fedf497f9d20331b964ab625, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960627546 2024-11-18T20:10:31,160 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b037ee2e23b4d168af7727b7d6d80a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960627560 2024-11-18T20:10:31,160 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec272de7c6ee4446be4998a9dcc7e5b2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960628790 2024-11-18T20:10:31,160 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting db586b7f9cf74423a2e7394035519a9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960628806 2024-11-18T20:10:31,198 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:31,198 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/7de6d575f4fa4e0d88572af846d5e5c3 is 50, key is test_row_0/A:col10/1731960629988/Put/seqid=0 2024-11-18T20:10:31,201 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:31,201 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/3477e0287b8a4995b21cfa8e91583cd3 is 50, key is test_row_0/C:col10/1731960629988/Put/seqid=0 2024-11-18T20:10:31,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:10:31,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742285_1461 (size=12104) 2024-11-18T20:10:31,274 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:10:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/7b2d3b3dc37340278564aedb0969eaa2 is 50, key is test_row_0/A:col10/1731960630164/Put/seqid=0 2024-11-18T20:10:31,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742286_1462 (size=12104) 2024-11-18T20:10:31,305 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/7de6d575f4fa4e0d88572af846d5e5c3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7de6d575f4fa4e0d88572af846d5e5c3 2024-11-18T20:10:31,322 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into 7de6d575f4fa4e0d88572af846d5e5c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:31,322 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:31,322 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=13, startTime=1731960631148; duration=0sec 2024-11-18T20:10:31,322 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:31,322 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:31,322 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:31,324 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:31,324 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:31,324 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,324 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/a2b133365c524d869f0bb48903d61d56, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d431c6e580cb4980adf6b73578e8da18, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/c89b26afa2b346b89480f555100af068] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=35.2 K 2024-11-18T20:10:31,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742287_1463 (size=12001) 2024-11-18T20:10:31,325 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2b133365c524d869f0bb48903d61d56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960627435 2024-11-18T20:10:31,327 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d431c6e580cb4980adf6b73578e8da18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960627560 2024-11-18T20:10:31,334 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89b26afa2b346b89480f555100af068, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960628806 2024-11-18T20:10:31,335 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/7b2d3b3dc37340278564aedb0969eaa2 2024-11-18T20:10:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/e3c42a3a0dce486598df04e5ad42d53c is 50, key is test_row_0/B:col10/1731960630164/Put/seqid=0 2024-11-18T20:10:31,386 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:31,386 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/be667ab04140400c86099c4cdc856d42 is 50, key is test_row_0/B:col10/1731960629988/Put/seqid=0 2024-11-18T20:10:31,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742288_1464 (size=12001) 2024-11-18T20:10:31,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742289_1465 (size=12104) 2024-11-18T20:10:31,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960691411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960691411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960691412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960691414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960691424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960691534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960691534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960691544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960691545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960691545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:31,719 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/3477e0287b8a4995b21cfa8e91583cd3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3477e0287b8a4995b21cfa8e91583cd3 2024-11-18T20:10:31,729 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into 3477e0287b8a4995b21cfa8e91583cd3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:31,729 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:31,729 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=13, startTime=1731960631148; duration=0sec 2024-11-18T20:10:31,729 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:31,729 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:31,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960691755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960691762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960691762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960691762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960691763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:31,829 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/e3c42a3a0dce486598df04e5ad42d53c 2024-11-18T20:10:31,840 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/be667ab04140400c86099c4cdc856d42 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/be667ab04140400c86099c4cdc856d42 2024-11-18T20:10:31,851 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into be667ab04140400c86099c4cdc856d42(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:31,851 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:31,851 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=13, startTime=1731960631148; duration=0sec 2024-11-18T20:10:31,852 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:31,852 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a4cbbba40a4e4a999b13c0b24028bd2b is 50, key is test_row_0/C:col10/1731960630164/Put/seqid=0 2024-11-18T20:10:31,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742290_1466 (size=12001) 2024-11-18T20:10:31,905 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a4cbbba40a4e4a999b13c0b24028bd2b 2024-11-18T20:10:31,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/7b2d3b3dc37340278564aedb0969eaa2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7b2d3b3dc37340278564aedb0969eaa2 2024-11-18T20:10:31,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7b2d3b3dc37340278564aedb0969eaa2, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:10:31,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/e3c42a3a0dce486598df04e5ad42d53c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/e3c42a3a0dce486598df04e5ad42d53c 2024-11-18T20:10:31,941 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/e3c42a3a0dce486598df04e5ad42d53c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:10:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a4cbbba40a4e4a999b13c0b24028bd2b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a4cbbba40a4e4a999b13c0b24028bd2b 2024-11-18T20:10:31,972 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a4cbbba40a4e4a999b13c0b24028bd2b, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:10:31,974 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 471b1d433f657c986c9e035c4a593a36 in 700ms, sequenceid=75, compaction requested=false 2024-11-18T20:10:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-18T20:10:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-18T20:10:31,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-18T20:10:31,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5640 sec 2024-11-18T20:10:31,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 4.5770 sec 2024-11-18T20:10:32,084 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:32,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:10:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:32,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/961086cd99ea471b819e527fd9187f95 is 50, key is test_row_0/A:col10/1731960632078/Put/seqid=0 2024-11-18T20:10:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742291_1467 (size=12001) 2024-11-18T20:10:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960692186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960692187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960692193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960692195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960692197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960692323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960692323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960692323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960692323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960692325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960692542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960692542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960692542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960692543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960692544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/961086cd99ea471b819e527fd9187f95 2024-11-18T20:10:32,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/0f69b4dd95f1461d90bd5cc147cdde33 is 50, key is test_row_0/B:col10/1731960632078/Put/seqid=0 2024-11-18T20:10:32,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742292_1468 (size=12001) 2024-11-18T20:10:32,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960692863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960692863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960692863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960692863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960692863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:33,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/0f69b4dd95f1461d90bd5cc147cdde33 2024-11-18T20:10:33,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/de1c1d6082304e969b9791db9c7adae8 is 50, key is test_row_0/C:col10/1731960632078/Put/seqid=0 2024-11-18T20:10:33,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742293_1469 (size=12001) 2024-11-18T20:10:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960693382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960693382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960693382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960693383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960693384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:33,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/de1c1d6082304e969b9791db9c7adae8 2024-11-18T20:10:33,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/961086cd99ea471b819e527fd9187f95 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/961086cd99ea471b819e527fd9187f95 2024-11-18T20:10:33,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/961086cd99ea471b819e527fd9187f95, entries=150, sequenceid=94, filesize=11.7 K 2024-11-18T20:10:33,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/0f69b4dd95f1461d90bd5cc147cdde33 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/0f69b4dd95f1461d90bd5cc147cdde33 2024-11-18T20:10:33,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/0f69b4dd95f1461d90bd5cc147cdde33, entries=150, sequenceid=94, filesize=11.7 K 2024-11-18T20:10:33,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/de1c1d6082304e969b9791db9c7adae8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/de1c1d6082304e969b9791db9c7adae8 2024-11-18T20:10:33,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/de1c1d6082304e969b9791db9c7adae8, entries=150, sequenceid=94, filesize=11.7 K 2024-11-18T20:10:33,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 471b1d433f657c986c9e035c4a593a36 in 1614ms, sequenceid=94, compaction requested=true 2024-11-18T20:10:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:33,698 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:33,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:33,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:33,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:33,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:33,699 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:33,706 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:33,706 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:33,706 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:33,706 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7de6d575f4fa4e0d88572af846d5e5c3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7b2d3b3dc37340278564aedb0969eaa2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/961086cd99ea471b819e527fd9187f95] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=35.3 K 2024-11-18T20:10:33,711 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de6d575f4fa4e0d88572af846d5e5c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960628806 2024-11-18T20:10:33,712 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:33,712 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:33,712 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:33,712 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/be667ab04140400c86099c4cdc856d42, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/e3c42a3a0dce486598df04e5ad42d53c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/0f69b4dd95f1461d90bd5cc147cdde33] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=35.3 K 2024-11-18T20:10:33,714 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2d3b3dc37340278564aedb0969eaa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960630144 2024-11-18T20:10:33,715 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting be667ab04140400c86099c4cdc856d42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960628806 2024-11-18T20:10:33,715 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 961086cd99ea471b819e527fd9187f95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960631365 2024-11-18T20:10:33,715 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c42a3a0dce486598df04e5ad42d53c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960630144 2024-11-18T20:10:33,722 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f69b4dd95f1461d90bd5cc147cdde33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960631365 2024-11-18T20:10:33,785 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:33,786 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b700df98ff9844ceb0822eda3f2fd86a is 50, key is test_row_0/A:col10/1731960632078/Put/seqid=0 2024-11-18T20:10:33,793 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:33,794 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/cab4af8f8be4402e94ac7b507852af86 is 50, key is test_row_0/B:col10/1731960632078/Put/seqid=0 2024-11-18T20:10:33,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742294_1470 (size=12207) 2024-11-18T20:10:33,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742295_1471 (size=12207) 2024-11-18T20:10:33,903 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/cab4af8f8be4402e94ac7b507852af86 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/cab4af8f8be4402e94ac7b507852af86 2024-11-18T20:10:33,921 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into cab4af8f8be4402e94ac7b507852af86(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:33,921 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:33,921 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=13, startTime=1731960633698; duration=0sec 2024-11-18T20:10:33,921 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:33,921 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:33,921 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:33,935 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:33,935 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:33,935 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:33,935 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3477e0287b8a4995b21cfa8e91583cd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a4cbbba40a4e4a999b13c0b24028bd2b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/de1c1d6082304e969b9791db9c7adae8] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=35.3 K 2024-11-18T20:10:33,937 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3477e0287b8a4995b21cfa8e91583cd3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960628806 2024-11-18T20:10:33,939 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a4cbbba40a4e4a999b13c0b24028bd2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960630144 2024-11-18T20:10:33,940 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting de1c1d6082304e969b9791db9c7adae8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960631365 2024-11-18T20:10:33,958 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:33,959 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/d22c20d6e9c940c9a6c88533900cf18b is 50, key is test_row_0/C:col10/1731960632078/Put/seqid=0 2024-11-18T20:10:34,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742296_1472 (size=12207) 2024-11-18T20:10:34,030 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/d22c20d6e9c940c9a6c88533900cf18b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/d22c20d6e9c940c9a6c88533900cf18b 2024-11-18T20:10:34,043 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into d22c20d6e9c940c9a6c88533900cf18b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:34,043 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:34,043 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=13, startTime=1731960633699; duration=0sec 2024-11-18T20:10:34,043 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:34,044 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:34,273 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b700df98ff9844ceb0822eda3f2fd86a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b700df98ff9844ceb0822eda3f2fd86a 2024-11-18T20:10:34,289 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into b700df98ff9844ceb0822eda3f2fd86a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:34,289 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:34,289 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=13, startTime=1731960633698; duration=0sec 2024-11-18T20:10:34,289 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:34,289 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:34,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:34,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:10:34,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b59385e78b9e4737bf9c81b5d68ef8e4 is 50, key is test_row_0/A:col10/1731960634405/Put/seqid=0 2024-11-18T20:10:34,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742297_1473 (size=12001) 2024-11-18T20:10:34,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b59385e78b9e4737bf9c81b5d68ef8e4 2024-11-18T20:10:34,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/6f40e7bbeb344bcb9b7a6984852455ee is 50, key is test_row_0/B:col10/1731960634405/Put/seqid=0 2024-11-18T20:10:34,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742298_1474 (size=12001) 2024-11-18T20:10:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960694569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960694569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960694580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960694600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960694605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960694716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960694718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960694719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960694736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960694736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960694935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960694935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960694940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960694956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960694956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/6f40e7bbeb344bcb9b7a6984852455ee 2024-11-18T20:10:35,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/baa554cb5a244214b3f6240a8604ef1c is 50, key is test_row_0/C:col10/1731960634405/Put/seqid=0 2024-11-18T20:10:35,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742299_1475 (size=12001) 2024-11-18T20:10:35,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/baa554cb5a244214b3f6240a8604ef1c 2024-11-18T20:10:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b59385e78b9e4737bf9c81b5d68ef8e4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b59385e78b9e4737bf9c81b5d68ef8e4 2024-11-18T20:10:35,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b59385e78b9e4737bf9c81b5d68ef8e4, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T20:10:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/6f40e7bbeb344bcb9b7a6984852455ee as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6f40e7bbeb344bcb9b7a6984852455ee 2024-11-18T20:10:35,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960695252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960695252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960695255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6f40e7bbeb344bcb9b7a6984852455ee, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T20:10:35,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/baa554cb5a244214b3f6240a8604ef1c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/baa554cb5a244214b3f6240a8604ef1c 2024-11-18T20:10:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960695276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960695279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/baa554cb5a244214b3f6240a8604ef1c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-18T20:10:35,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 471b1d433f657c986c9e035c4a593a36 in 923ms, sequenceid=119, compaction requested=false 2024-11-18T20:10:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:10:35,602 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:35,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-18T20:10:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:10:35,605 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:35,606 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:35,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:10:35,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:10:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:35,763 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:10:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:35,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b100f90ee210417eae2c0073191db16b is 50, key is test_row_0/A:col10/1731960634444/Put/seqid=0 2024-11-18T20:10:35,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742300_1476 (size=12101) 2024-11-18T20:10:35,839 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b100f90ee210417eae2c0073191db16b 2024-11-18T20:10:35,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960695831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960695836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960695838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0 is 50, key is test_row_0/B:col10/1731960634444/Put/seqid=0 2024-11-18T20:10:35,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960695846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960695849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:10:35,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742301_1477 (size=12101) 2024-11-18T20:10:35,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960695952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960695975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960695977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960695979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960695979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960696172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960696192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960696192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960696192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960696202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:10:36,335 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0 2024-11-18T20:10:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/27bbea9977bc4d2b938281f169821db3 is 50, key is test_row_0/C:col10/1731960634444/Put/seqid=0 2024-11-18T20:10:36,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742302_1478 (size=12101) 2024-11-18T20:10:36,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/27bbea9977bc4d2b938281f169821db3 2024-11-18T20:10:36,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960696483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/b100f90ee210417eae2c0073191db16b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b100f90ee210417eae2c0073191db16b 2024-11-18T20:10:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960696502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960696502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960696503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960696522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:36,537 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b100f90ee210417eae2c0073191db16b, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T20:10:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0 2024-11-18T20:10:36,551 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T20:10:36,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/27bbea9977bc4d2b938281f169821db3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/27bbea9977bc4d2b938281f169821db3 2024-11-18T20:10:36,572 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/27bbea9977bc4d2b938281f169821db3, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T20:10:36,573 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 471b1d433f657c986c9e035c4a593a36 in 810ms, sequenceid=134, compaction requested=true 2024-11-18T20:10:36,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:36,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:36,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-18T20:10:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-18T20:10:36,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-18T20:10:36,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 970 msec 2024-11-18T20:10:36,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 979 msec 2024-11-18T20:10:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:10:36,746 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:36,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-18T20:10:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:10:36,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:36,762 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:36,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:10:36,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:10:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:36,920 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:10:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/c7050d61f7e643ed8d605d67fcaacbe1 is 50, key is test_row_0/A:col10/1731960635830/Put/seqid=0 2024-11-18T20:10:36,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742303_1479 (size=12151) 2024-11-18T20:10:37,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:10:37,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960697057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960697071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960697072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960697081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960697081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960697187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960697194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960697204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960697204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960697205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:10:37,397 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/c7050d61f7e643ed8d605d67fcaacbe1 2024-11-18T20:10:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960697412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960697412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/edd59e8768364cd18ea5c7ca9f252125 is 50, key is test_row_0/B:col10/1731960635830/Put/seqid=0 2024-11-18T20:10:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960697428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742304_1480 (size=12151) 2024-11-18T20:10:37,442 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/edd59e8768364cd18ea5c7ca9f252125 2024-11-18T20:10:37,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960697435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960697438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/506d09b836794784864cfee4ce62b3c4 is 50, key is test_row_0/C:col10/1731960635830/Put/seqid=0 2024-11-18T20:10:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742305_1481 (size=12151) 2024-11-18T20:10:37,456 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/506d09b836794784864cfee4ce62b3c4 2024-11-18T20:10:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/c7050d61f7e643ed8d605d67fcaacbe1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/c7050d61f7e643ed8d605d67fcaacbe1 2024-11-18T20:10:37,469 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/c7050d61f7e643ed8d605d67fcaacbe1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:10:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/edd59e8768364cd18ea5c7ca9f252125 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/edd59e8768364cd18ea5c7ca9f252125 2024-11-18T20:10:37,478 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/edd59e8768364cd18ea5c7ca9f252125, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:10:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/506d09b836794784864cfee4ce62b3c4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/506d09b836794784864cfee4ce62b3c4 2024-11-18T20:10:37,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/506d09b836794784864cfee4ce62b3c4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:10:37,485 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 471b1d433f657c986c9e035c4a593a36 in 566ms, sequenceid=155, compaction requested=true 2024-11-18T20:10:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-18T20:10:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-18T20:10:37,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-18T20:10:37,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 728 msec 2024-11-18T20:10:37,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 756 msec 2024-11-18T20:10:37,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:37,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:10:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:37,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/fc6b8fe712f2410fb363d94398f45fa3 is 50, key is test_row_0/A:col10/1731960637066/Put/seqid=0 2024-11-18T20:10:37,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742306_1482 (size=14541) 2024-11-18T20:10:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960697803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960697805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960697807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960697809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960697812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:10:37,895 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:37,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-18T20:10:37,897 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:37,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:37,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:10:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960697932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960697932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960697932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960697932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960697933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:10:38,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:10:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960698152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960698152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960698153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960698153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960698154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/fc6b8fe712f2410fb363d94398f45fa3 2024-11-18T20:10:38,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:10:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:38,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:10:38,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/dd161620146c4a71a2578e9a420d61ed is 50, key is test_row_0/B:col10/1731960637066/Put/seqid=0 2024-11-18T20:10:38,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742307_1483 (size=12151) 2024-11-18T20:10:38,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:10:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960698462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960698462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960698462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960698462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960698472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:10:38,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:10:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/dd161620146c4a71a2578e9a420d61ed 2024-11-18T20:10:38,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:10:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:38,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/ced90f472f624eeb99cb361e6cb057d7 is 50, key is test_row_0/C:col10/1731960637066/Put/seqid=0 2024-11-18T20:10:38,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742308_1484 (size=12151) 2024-11-18T20:10:38,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/ced90f472f624eeb99cb361e6cb057d7 2024-11-18T20:10:38,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/fc6b8fe712f2410fb363d94398f45fa3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/fc6b8fe712f2410fb363d94398f45fa3 2024-11-18T20:10:38,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/fc6b8fe712f2410fb363d94398f45fa3, entries=200, sequenceid=171, filesize=14.2 K 2024-11-18T20:10:38,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/dd161620146c4a71a2578e9a420d61ed as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/dd161620146c4a71a2578e9a420d61ed 2024-11-18T20:10:38,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/dd161620146c4a71a2578e9a420d61ed, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:10:38,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/ced90f472f624eeb99cb361e6cb057d7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/ced90f472f624eeb99cb361e6cb057d7 2024-11-18T20:10:38,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/ced90f472f624eeb99cb361e6cb057d7, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:10:38,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 471b1d433f657c986c9e035c4a593a36 in 1053ms, sequenceid=171, compaction requested=true 2024-11-18T20:10:38,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:38,796 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T20:10:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:38,797 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T20:10:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:38,800 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63001 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T20:10:38,800 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:38,800 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,800 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b700df98ff9844ceb0822eda3f2fd86a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b59385e78b9e4737bf9c81b5d68ef8e4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b100f90ee210417eae2c0073191db16b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/c7050d61f7e643ed8d605d67fcaacbe1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/fc6b8fe712f2410fb363d94398f45fa3] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=61.5 K 2024-11-18T20:10:38,801 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b700df98ff9844ceb0822eda3f2fd86a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960631365 2024-11-18T20:10:38,801 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T20:10:38,801 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:38,801 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,801 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/cab4af8f8be4402e94ac7b507852af86, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6f40e7bbeb344bcb9b7a6984852455ee, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/edd59e8768364cd18ea5c7ca9f252125, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/dd161620146c4a71a2578e9a420d61ed] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=59.2 K 2024-11-18T20:10:38,801 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59385e78b9e4737bf9c81b5d68ef8e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731960632188 2024-11-18T20:10:38,802 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting cab4af8f8be4402e94ac7b507852af86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960631365 2024-11-18T20:10:38,802 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b100f90ee210417eae2c0073191db16b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960634444 2024-11-18T20:10:38,802 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f40e7bbeb344bcb9b7a6984852455ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731960632188 2024-11-18T20:10:38,802 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7050d61f7e643ed8d605d67fcaacbe1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960635827 2024-11-18T20:10:38,802 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d3c66ec5464f2e9d2d4e7efb7b9ea0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960634444 2024-11-18T20:10:38,802 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6b8fe712f2410fb363d94398f45fa3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960637043 2024-11-18T20:10:38,803 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting edd59e8768364cd18ea5c7ca9f252125, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960635827 2024-11-18T20:10:38,803 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting dd161620146c4a71a2578e9a420d61ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960637043 2024-11-18T20:10:38,828 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#411 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:38,829 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/61ecd95f04d548bda3d073c7337ce86d is 50, key is test_row_0/A:col10/1731960637066/Put/seqid=0 2024-11-18T20:10:38,837 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:38,838 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/878b55cb22dc4963a3ad82b746da59b4 is 50, key is test_row_0/B:col10/1731960637066/Put/seqid=0 2024-11-18T20:10:38,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:10:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:38,863 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:10:38,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742309_1485 (size=12527) 2024-11-18T20:10:38,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742310_1486 (size=12527) 2024-11-18T20:10:38,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/8b8601fbf10d4cae8646b2e9b07c2703 is 50, key is test_row_0/A:col10/1731960637807/Put/seqid=0 2024-11-18T20:10:38,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742311_1487 (size=12151) 2024-11-18T20:10:38,944 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/8b8601fbf10d4cae8646b2e9b07c2703 2024-11-18T20:10:38,959 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/878b55cb22dc4963a3ad82b746da59b4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/878b55cb22dc4963a3ad82b746da59b4 2024-11-18T20:10:38,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/6a1f40809e644819825dded429806369 is 50, key is test_row_0/B:col10/1731960637807/Put/seqid=0 2024-11-18T20:10:39,020 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into 878b55cb22dc4963a3ad82b746da59b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:39,020 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:39,020 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=11, startTime=1731960638797; duration=0sec 2024-11-18T20:10:39,020 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:39,020 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:39,020 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-18T20:10:39,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742312_1488 (size=12151) 2024-11-18T20:10:39,035 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-18T20:10:39,035 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:39,035 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:39,036 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/d22c20d6e9c940c9a6c88533900cf18b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/baa554cb5a244214b3f6240a8604ef1c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/27bbea9977bc4d2b938281f169821db3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/506d09b836794784864cfee4ce62b3c4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/ced90f472f624eeb99cb361e6cb057d7] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=59.2 K 2024-11-18T20:10:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:10:39,043 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d22c20d6e9c940c9a6c88533900cf18b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960631365 2024-11-18T20:10:39,044 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/6a1f40809e644819825dded429806369 2024-11-18T20:10:39,045 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting baa554cb5a244214b3f6240a8604ef1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731960632188 2024-11-18T20:10:39,047 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 27bbea9977bc4d2b938281f169821db3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731960634444 2024-11-18T20:10:39,052 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 506d09b836794784864cfee4ce62b3c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960635827 2024-11-18T20:10:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960699029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960699032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,063 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ced90f472f624eeb99cb361e6cb057d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960637043 2024-11-18T20:10:39,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960699050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960699052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960699057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/843e043dfcf844ca92389a8bc51bf0bf is 50, key is test_row_0/C:col10/1731960637807/Put/seqid=0 2024-11-18T20:10:39,142 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:39,143 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/cb0ee776e7ff4591875d13dbfaddd812 is 50, key is test_row_0/C:col10/1731960637066/Put/seqid=0 2024-11-18T20:10:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742313_1489 (size=12151) 2024-11-18T20:10:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742314_1490 (size=12527) 2024-11-18T20:10:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960699163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960699162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960699184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960699185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960699186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,278 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/61ecd95f04d548bda3d073c7337ce86d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/61ecd95f04d548bda3d073c7337ce86d 2024-11-18T20:10:39,286 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into 61ecd95f04d548bda3d073c7337ce86d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:39,286 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:39,286 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=11, startTime=1731960638796; duration=0sec 2024-11-18T20:10:39,286 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:39,286 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960699383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960699394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960699412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960699412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960699412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,547 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/843e043dfcf844ca92389a8bc51bf0bf 2024-11-18T20:10:39,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/8b8601fbf10d4cae8646b2e9b07c2703 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/8b8601fbf10d4cae8646b2e9b07c2703 2024-11-18T20:10:39,569 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/8b8601fbf10d4cae8646b2e9b07c2703, entries=150, sequenceid=191, filesize=11.9 K 2024-11-18T20:10:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/6a1f40809e644819825dded429806369 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6a1f40809e644819825dded429806369 2024-11-18T20:10:39,576 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/cb0ee776e7ff4591875d13dbfaddd812 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/cb0ee776e7ff4591875d13dbfaddd812 2024-11-18T20:10:39,578 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6a1f40809e644819825dded429806369, entries=150, sequenceid=191, filesize=11.9 K 2024-11-18T20:10:39,580 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into cb0ee776e7ff4591875d13dbfaddd812(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:39,581 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:39,581 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=11, startTime=1731960638797; duration=0sec 2024-11-18T20:10:39,581 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:39,581 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:39,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/843e043dfcf844ca92389a8bc51bf0bf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/843e043dfcf844ca92389a8bc51bf0bf 2024-11-18T20:10:39,587 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/843e043dfcf844ca92389a8bc51bf0bf, entries=150, sequenceid=191, filesize=11.9 K 2024-11-18T20:10:39,588 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 471b1d433f657c986c9e035c4a593a36 in 725ms, sequenceid=191, compaction requested=false 2024-11-18T20:10:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-18T20:10:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-18T20:10:39,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-18T20:10:39,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6920 sec 2024-11-18T20:10:39,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6980 sec 2024-11-18T20:10:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:39,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:10:39,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/ba12d14499a340e082e3bb376fcbe1c5 is 50, key is test_row_0/A:col10/1731960639050/Put/seqid=0 2024-11-18T20:10:39,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742315_1491 (size=14541) 2024-11-18T20:10:39,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960699734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960699740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960699741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960699741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960699741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960699852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960699852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960699852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960699852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960699852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:39,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:10:40,057 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:40,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-18T20:10:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:10:40,059 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:40,059 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:40,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960700062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960700062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960700074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960700074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960700075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/ba12d14499a340e082e3bb376fcbe1c5 2024-11-18T20:10:40,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b019675916a94b4cbf4524392af0463a is 50, key is test_row_0/B:col10/1731960639050/Put/seqid=0 2024-11-18T20:10:40,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742316_1492 (size=12151) 2024-11-18T20:10:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:10:40,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:10:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960700372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:10:40,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960700372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:10:40,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960700392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960700392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960700393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:10:40,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:40,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b019675916a94b4cbf4524392af0463a 2024-11-18T20:10:40,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/9c5556f72cb049acb62029d98b21d326 is 50, key is test_row_0/C:col10/1731960639050/Put/seqid=0 2024-11-18T20:10:40,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742317_1493 (size=12151) 2024-11-18T20:10:40,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/9c5556f72cb049acb62029d98b21d326 2024-11-18T20:10:40,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/ba12d14499a340e082e3bb376fcbe1c5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ba12d14499a340e082e3bb376fcbe1c5 2024-11-18T20:10:40,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ba12d14499a340e082e3bb376fcbe1c5, entries=200, sequenceid=211, filesize=14.2 K 2024-11-18T20:10:40,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b019675916a94b4cbf4524392af0463a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b019675916a94b4cbf4524392af0463a 2024-11-18T20:10:40,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b019675916a94b4cbf4524392af0463a, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T20:10:40,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/9c5556f72cb049acb62029d98b21d326 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/9c5556f72cb049acb62029d98b21d326 2024-11-18T20:10:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:10:40,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:10:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:40,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/9c5556f72cb049acb62029d98b21d326, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T20:10:40,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 471b1d433f657c986c9e035c4a593a36 in 1008ms, sequenceid=211, compaction requested=true 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:40,706 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:40,706 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:40,715 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:40,715 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:40,716 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:40,716 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:40,716 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,716 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/878b55cb22dc4963a3ad82b746da59b4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6a1f40809e644819825dded429806369, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b019675916a94b4cbf4524392af0463a] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=36.0 K 2024-11-18T20:10:40,716 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,716 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/61ecd95f04d548bda3d073c7337ce86d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/8b8601fbf10d4cae8646b2e9b07c2703, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ba12d14499a340e082e3bb376fcbe1c5] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=38.3 K 2024-11-18T20:10:40,719 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ecd95f04d548bda3d073c7337ce86d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960637043 2024-11-18T20:10:40,720 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8601fbf10d4cae8646b2e9b07c2703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731960637804 2024-11-18T20:10:40,723 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 878b55cb22dc4963a3ad82b746da59b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960637043 2024-11-18T20:10:40,724 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba12d14499a340e082e3bb376fcbe1c5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960639026 2024-11-18T20:10:40,727 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a1f40809e644819825dded429806369, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731960637804 2024-11-18T20:10:40,729 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b019675916a94b4cbf4524392af0463a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960639026 2024-11-18T20:10:40,762 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:40,763 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1cda25c544b4424f8134c1ad5d27d81c is 50, key is test_row_0/A:col10/1731960639050/Put/seqid=0 2024-11-18T20:10:40,766 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:40,766 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d0bccba6b0024258b2b1b483f46fcd5e is 50, key is test_row_0/B:col10/1731960639050/Put/seqid=0 2024-11-18T20:10:40,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742319_1495 (size=12629) 2024-11-18T20:10:40,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742318_1494 (size=12629) 2024-11-18T20:10:40,815 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d0bccba6b0024258b2b1b483f46fcd5e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0bccba6b0024258b2b1b483f46fcd5e 2024-11-18T20:10:40,829 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into d0bccba6b0024258b2b1b483f46fcd5e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:40,829 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:40,829 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=13, startTime=1731960640706; duration=0sec 2024-11-18T20:10:40,830 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:40,830 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:40,830 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:40,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:40,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:40,831 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,831 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/cb0ee776e7ff4591875d13dbfaddd812, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/843e043dfcf844ca92389a8bc51bf0bf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/9c5556f72cb049acb62029d98b21d326] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=36.0 K 2024-11-18T20:10:40,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0ee776e7ff4591875d13dbfaddd812, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960637043 2024-11-18T20:10:40,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 843e043dfcf844ca92389a8bc51bf0bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731960637804 2024-11-18T20:10:40,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5556f72cb049acb62029d98b21d326, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960639026 2024-11-18T20:10:40,847 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:40,848 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/18b4377ce32046f0a5790d7d0a0769ba is 50, key is test_row_0/C:col10/1731960639050/Put/seqid=0 2024-11-18T20:10:40,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:10:40,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:40,852 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-18T20:10:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742320_1496 (size=12629) 2024-11-18T20:10:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/2a6ed24c5df543478d4786a426cb35b7 is 50, key is test_row_0/A:col10/1731960639728/Put/seqid=0 2024-11-18T20:10:40,872 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/18b4377ce32046f0a5790d7d0a0769ba as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/18b4377ce32046f0a5790d7d0a0769ba 2024-11-18T20:10:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742321_1497 (size=12151) 2024-11-18T20:10:40,884 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into 18b4377ce32046f0a5790d7d0a0769ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:40,884 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:40,884 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=13, startTime=1731960640706; duration=0sec 2024-11-18T20:10:40,884 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:40,884 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:40,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:40,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960700911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960700913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960700919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960700920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960700921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960701032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960701032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960701032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960701032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960701042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:10:41,218 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1cda25c544b4424f8134c1ad5d27d81c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1cda25c544b4424f8134c1ad5d27d81c 2024-11-18T20:10:41,222 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into 1cda25c544b4424f8134c1ad5d27d81c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:41,222 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:41,222 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=13, startTime=1731960640706; duration=0sec 2024-11-18T20:10:41,222 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:41,222 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:41,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960701241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960701242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960701242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960701242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960701252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,273 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/2a6ed24c5df543478d4786a426cb35b7 2024-11-18T20:10:41,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b209937d2305421e9c2c317f8f9a0707 is 50, key is test_row_0/B:col10/1731960639728/Put/seqid=0 2024-11-18T20:10:41,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742322_1498 (size=12151) 2024-11-18T20:10:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960701552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960701552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960701553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960701562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960701562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:41,687 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b209937d2305421e9c2c317f8f9a0707 2024-11-18T20:10:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/c59aed260dee44e9ba2452806c9946fa is 50, key is test_row_0/C:col10/1731960639728/Put/seqid=0 2024-11-18T20:10:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742323_1499 (size=12151) 2024-11-18T20:10:41,723 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/c59aed260dee44e9ba2452806c9946fa 2024-11-18T20:10:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/2a6ed24c5df543478d4786a426cb35b7 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/2a6ed24c5df543478d4786a426cb35b7 2024-11-18T20:10:41,735 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/2a6ed24c5df543478d4786a426cb35b7, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T20:10:41,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b209937d2305421e9c2c317f8f9a0707 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b209937d2305421e9c2c317f8f9a0707 2024-11-18T20:10:41,740 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b209937d2305421e9c2c317f8f9a0707, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T20:10:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/c59aed260dee44e9ba2452806c9946fa as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/c59aed260dee44e9ba2452806c9946fa 2024-11-18T20:10:41,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/c59aed260dee44e9ba2452806c9946fa, entries=150, sequenceid=231, filesize=11.9 K 2024-11-18T20:10:41,747 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 471b1d433f657c986c9e035c4a593a36 in 896ms, sequenceid=231, compaction requested=false 2024-11-18T20:10:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-18T20:10:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-18T20:10:41,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-18T20:10:41,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6900 sec 2024-11-18T20:10:41,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6980 sec 2024-11-18T20:10:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:42,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:10:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:42,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/d0185f0dc00f47fe8554118f6c7c6411 is 50, key is test_row_0/A:col10/1731960640909/Put/seqid=0 2024-11-18T20:10:42,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742324_1500 (size=14541) 2024-11-18T20:10:42,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/d0185f0dc00f47fe8554118f6c7c6411 2024-11-18T20:10:42,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/4c5a7c2dfa4d4ed78fc6562302b2136d is 50, key is test_row_0/B:col10/1731960640909/Put/seqid=0 2024-11-18T20:10:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960702095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960702097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960702099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960702100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960702108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742325_1501 (size=12151) 2024-11-18T20:10:42,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/4c5a7c2dfa4d4ed78fc6562302b2136d 2024-11-18T20:10:42,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a35d686054b84b3fb8bf8c3cfdd4df5c is 50, key is test_row_0/C:col10/1731960640909/Put/seqid=0 2024-11-18T20:10:42,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742326_1502 (size=12151) 2024-11-18T20:10:42,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a35d686054b84b3fb8bf8c3cfdd4df5c 2024-11-18T20:10:42,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/d0185f0dc00f47fe8554118f6c7c6411 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/d0185f0dc00f47fe8554118f6c7c6411 2024-11-18T20:10:42,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/d0185f0dc00f47fe8554118f6c7c6411, entries=200, sequenceid=251, filesize=14.2 K 2024-11-18T20:10:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/4c5a7c2dfa4d4ed78fc6562302b2136d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/4c5a7c2dfa4d4ed78fc6562302b2136d 2024-11-18T20:10:42,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/4c5a7c2dfa4d4ed78fc6562302b2136d, entries=150, sequenceid=251, filesize=11.9 K 2024-11-18T20:10:42,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a35d686054b84b3fb8bf8c3cfdd4df5c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a35d686054b84b3fb8bf8c3cfdd4df5c 2024-11-18T20:10:42,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a35d686054b84b3fb8bf8c3cfdd4df5c, entries=150, sequenceid=251, filesize=11.9 K 2024-11-18T20:10:42,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 471b1d433f657c986c9e035c4a593a36 in 107ms, sequenceid=251, compaction requested=true 2024-11-18T20:10:42,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:42,176 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:42,176 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:42,178 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:42,178 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:42,179 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,179 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1cda25c544b4424f8134c1ad5d27d81c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/2a6ed24c5df543478d4786a426cb35b7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/d0185f0dc00f47fe8554118f6c7c6411] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=38.4 K 2024-11-18T20:10:42,179 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:42,179 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:42,179 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,179 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0bccba6b0024258b2b1b483f46fcd5e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b209937d2305421e9c2c317f8f9a0707, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/4c5a7c2dfa4d4ed78fc6562302b2136d] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=36.1 K 2024-11-18T20:10:42,180 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d0bccba6b0024258b2b1b483f46fcd5e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960639026 2024-11-18T20:10:42,180 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cda25c544b4424f8134c1ad5d27d81c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960639026 2024-11-18T20:10:42,180 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b209937d2305421e9c2c317f8f9a0707, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731960639728 2024-11-18T20:10:42,180 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6ed24c5df543478d4786a426cb35b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731960639728 2024-11-18T20:10:42,181 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c5a7c2dfa4d4ed78fc6562302b2136d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960640909 2024-11-18T20:10:42,181 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0185f0dc00f47fe8554118f6c7c6411, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960640909 2024-11-18T20:10:42,205 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:42,206 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/04e2c7ac4047450db5f82f76e1650872 is 50, key is test_row_0/B:col10/1731960640909/Put/seqid=0 2024-11-18T20:10:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:10:42,212 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:42,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-18T20:10:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:42,215 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:42,216 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:42,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:42,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:42,217 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/02304e647e2a46d18bc4508dd0dbcc0f is 50, key is test_row_0/A:col10/1731960640909/Put/seqid=0 2024-11-18T20:10:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:42,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:10:42,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/0fb4dfd25ca540ee8f608c670e596809 is 50, key is test_row_0/A:col10/1731960642214/Put/seqid=0 2024-11-18T20:10:42,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742327_1503 (size=12731) 2024-11-18T20:10:42,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742328_1504 (size=12731) 2024-11-18T20:10:42,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742329_1505 (size=12301) 2024-11-18T20:10:42,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960702285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960702286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960702287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960702288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960702291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:42,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:10:42,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:42,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960702402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960702402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960702402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960702402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960702412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:10:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:42,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960702612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960702622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960702623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960702623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960702631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,657 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/04e2c7ac4047450db5f82f76e1650872 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/04e2c7ac4047450db5f82f76e1650872 2024-11-18T20:10:42,657 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/02304e647e2a46d18bc4508dd0dbcc0f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/02304e647e2a46d18bc4508dd0dbcc0f 2024-11-18T20:10:42,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/0fb4dfd25ca540ee8f608c670e596809 2024-11-18T20:10:42,662 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into 02304e647e2a46d18bc4508dd0dbcc0f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:42,662 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into 04e2c7ac4047450db5f82f76e1650872(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:42,662 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=13, startTime=1731960642176; duration=0sec 2024-11-18T20:10:42,662 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=13, startTime=1731960642176; duration=0sec 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:42,662 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:42,664 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:42,664 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:42,664 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,664 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/18b4377ce32046f0a5790d7d0a0769ba, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/c59aed260dee44e9ba2452806c9946fa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a35d686054b84b3fb8bf8c3cfdd4df5c] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=36.1 K 2024-11-18T20:10:42,664 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b4377ce32046f0a5790d7d0a0769ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960639026 2024-11-18T20:10:42,664 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting c59aed260dee44e9ba2452806c9946fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731960639728 2024-11-18T20:10:42,665 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a35d686054b84b3fb8bf8c3cfdd4df5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960640909 2024-11-18T20:10:42,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/52d682c1138e4af18db9fbc84afce282 is 50, key is test_row_0/B:col10/1731960642214/Put/seqid=0 2024-11-18T20:10:42,672 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:42,673 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/51954da6d4314c7c8d70754d123f6451 is 50, key is test_row_0/C:col10/1731960640909/Put/seqid=0 2024-11-18T20:10:42,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:10:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742330_1506 (size=12301) 2024-11-18T20:10:42,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742331_1507 (size=12731) 2024-11-18T20:10:42,711 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/51954da6d4314c7c8d70754d123f6451 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/51954da6d4314c7c8d70754d123f6451 2024-11-18T20:10:42,721 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into 51954da6d4314c7c8d70754d123f6451(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:42,721 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:42,721 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=13, startTime=1731960642176; duration=0sec 2024-11-18T20:10:42,722 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:42,722 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:42,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:10:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:42,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960702922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960702932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960702932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960702932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960702952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:42,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:10:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:42,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:43,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/52d682c1138e4af18db9fbc84afce282 2024-11-18T20:10:43,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/63699205ca204a7c8701a6fc2248cfba is 50, key is test_row_0/C:col10/1731960642214/Put/seqid=0 2024-11-18T20:10:43,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742332_1508 (size=12301) 2024-11-18T20:10:43,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/63699205ca204a7c8701a6fc2248cfba 2024-11-18T20:10:43,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/0fb4dfd25ca540ee8f608c670e596809 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/0fb4dfd25ca540ee8f608c670e596809 2024-11-18T20:10:43,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/0fb4dfd25ca540ee8f608c670e596809, entries=150, sequenceid=271, filesize=12.0 K 2024-11-18T20:10:43,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/52d682c1138e4af18db9fbc84afce282 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/52d682c1138e4af18db9fbc84afce282 2024-11-18T20:10:43,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/52d682c1138e4af18db9fbc84afce282, entries=150, sequenceid=271, filesize=12.0 K 2024-11-18T20:10:43,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/63699205ca204a7c8701a6fc2248cfba as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/63699205ca204a7c8701a6fc2248cfba 2024-11-18T20:10:43,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/63699205ca204a7c8701a6fc2248cfba, entries=150, sequenceid=271, filesize=12.0 K 2024-11-18T20:10:43,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 471b1d433f657c986c9e035c4a593a36 in 891ms, sequenceid=271, compaction requested=false 2024-11-18T20:10:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:43,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:10:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:43,140 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:10:43,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/43cee4172e9a4164995024d2e6d3d384 is 50, key is test_row_0/A:col10/1731960642287/Put/seqid=0 2024-11-18T20:10:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742333_1509 (size=12301) 2024-11-18T20:10:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:43,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960703475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960703479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960703480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960703484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960703488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,572 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/43cee4172e9a4164995024d2e6d3d384 2024-11-18T20:10:43,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/861b773a93fb48088617bcbb6b8177ec is 50, key is test_row_0/B:col10/1731960642287/Put/seqid=0 2024-11-18T20:10:43,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960703592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960703602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960703602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960703602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960703602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742334_1510 (size=12301) 2024-11-18T20:10:43,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960703812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960703822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960703822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960703822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960703822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,036 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/861b773a93fb48088617bcbb6b8177ec 2024-11-18T20:10:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/3c3445b972f142f98aa3cbe82a2101c4 is 50, key is test_row_0/C:col10/1731960642287/Put/seqid=0 2024-11-18T20:10:44,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742335_1511 (size=12301) 2024-11-18T20:10:44,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960704121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960704132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960704132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960704132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960704132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:44,453 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/3c3445b972f142f98aa3cbe82a2101c4 2024-11-18T20:10:44,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/43cee4172e9a4164995024d2e6d3d384 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/43cee4172e9a4164995024d2e6d3d384 2024-11-18T20:10:44,460 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/43cee4172e9a4164995024d2e6d3d384, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T20:10:44,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/861b773a93fb48088617bcbb6b8177ec as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/861b773a93fb48088617bcbb6b8177ec 2024-11-18T20:10:44,466 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/861b773a93fb48088617bcbb6b8177ec, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T20:10:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/3c3445b972f142f98aa3cbe82a2101c4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3c3445b972f142f98aa3cbe82a2101c4 2024-11-18T20:10:44,471 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3c3445b972f142f98aa3cbe82a2101c4, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T20:10:44,472 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 471b1d433f657c986c9e035c4a593a36 in 1332ms, sequenceid=291, compaction requested=true 2024-11-18T20:10:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-18T20:10:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-18T20:10:44,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-18T20:10:44,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2570 sec 2024-11-18T20:10:44,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.2620 sec 2024-11-18T20:10:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:44,647 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:44,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:10:44,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1d92dcd46caa4c6da5aea82ce483dfa2 is 50, key is test_row_0/A:col10/1731960643486/Put/seqid=0 2024-11-18T20:10:44,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742336_1512 (size=14741) 2024-11-18T20:10:44,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1d92dcd46caa4c6da5aea82ce483dfa2 2024-11-18T20:10:44,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960704671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b453a1c1783249568f60a0ca2a56ef16 is 50, key is test_row_0/B:col10/1731960643486/Put/seqid=0 2024-11-18T20:10:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960704677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960704678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960704679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960704680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742337_1513 (size=12301) 2024-11-18T20:10:44,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b453a1c1783249568f60a0ca2a56ef16 2024-11-18T20:10:44,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/947c445c67214619bb37652dc267f46d is 50, key is test_row_0/C:col10/1731960643486/Put/seqid=0 2024-11-18T20:10:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742338_1514 (size=12301) 2024-11-18T20:10:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960704791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960704792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960704792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960704802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960704802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960705001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960705001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960705002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960705012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960705012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/947c445c67214619bb37652dc267f46d 2024-11-18T20:10:45,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1d92dcd46caa4c6da5aea82ce483dfa2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d92dcd46caa4c6da5aea82ce483dfa2 2024-11-18T20:10:45,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d92dcd46caa4c6da5aea82ce483dfa2, entries=200, sequenceid=311, filesize=14.4 K 2024-11-18T20:10:45,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/b453a1c1783249568f60a0ca2a56ef16 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b453a1c1783249568f60a0ca2a56ef16 2024-11-18T20:10:45,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b453a1c1783249568f60a0ca2a56ef16, entries=150, sequenceid=311, filesize=12.0 K 2024-11-18T20:10:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/947c445c67214619bb37652dc267f46d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/947c445c67214619bb37652dc267f46d 2024-11-18T20:10:45,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/947c445c67214619bb37652dc267f46d, entries=150, sequenceid=311, filesize=12.0 K 2024-11-18T20:10:45,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 471b1d433f657c986c9e035c4a593a36 in 518ms, sequenceid=311, compaction requested=true 2024-11-18T20:10:45,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:45,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:45,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:45,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:45,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:45,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:45,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T20:10:45,166 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:45,166 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:45,168 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:45,168 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52074 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:45,168 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:45,168 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:45,168 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:45,168 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:45,169 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/51954da6d4314c7c8d70754d123f6451, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/63699205ca204a7c8701a6fc2248cfba, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3c3445b972f142f98aa3cbe82a2101c4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/947c445c67214619bb37652dc267f46d] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=48.5 K 2024-11-18T20:10:45,169 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51954da6d4314c7c8d70754d123f6451, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960640909 2024-11-18T20:10:45,169 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/02304e647e2a46d18bc4508dd0dbcc0f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/0fb4dfd25ca540ee8f608c670e596809, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/43cee4172e9a4164995024d2e6d3d384, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d92dcd46caa4c6da5aea82ce483dfa2] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=50.9 K 2024-11-18T20:10:45,170 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 02304e647e2a46d18bc4508dd0dbcc0f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960640909 2024-11-18T20:10:45,170 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63699205ca204a7c8701a6fc2248cfba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731960642099 2024-11-18T20:10:45,175 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3445b972f142f98aa3cbe82a2101c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960642285 2024-11-18T20:10:45,176 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947c445c67214619bb37652dc267f46d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960643476 2024-11-18T20:10:45,176 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb4dfd25ca540ee8f608c670e596809, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731960642099 2024-11-18T20:10:45,176 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 43cee4172e9a4164995024d2e6d3d384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960642285 2024-11-18T20:10:45,178 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d92dcd46caa4c6da5aea82ce483dfa2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960643472 2024-11-18T20:10:45,198 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:45,198 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/4056abc5a51c45d4a1291858bddce1e4 is 50, key is test_row_0/C:col10/1731960643486/Put/seqid=0 2024-11-18T20:10:45,234 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:45,234 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1d0b3c2989164a8b8f28349a39bd1584 is 50, key is test_row_0/A:col10/1731960643486/Put/seqid=0 2024-11-18T20:10:45,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742339_1515 (size=13017) 2024-11-18T20:10:45,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742340_1516 (size=13017) 2024-11-18T20:10:45,263 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/4056abc5a51c45d4a1291858bddce1e4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/4056abc5a51c45d4a1291858bddce1e4 2024-11-18T20:10:45,264 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/1d0b3c2989164a8b8f28349a39bd1584 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d0b3c2989164a8b8f28349a39bd1584 2024-11-18T20:10:45,266 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into 4056abc5a51c45d4a1291858bddce1e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:45,267 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:45,267 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=12, startTime=1731960645166; duration=0sec 2024-11-18T20:10:45,267 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:45,267 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:45,267 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:10:45,268 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:10:45,268 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:45,268 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:45,268 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/04e2c7ac4047450db5f82f76e1650872, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/52d682c1138e4af18db9fbc84afce282, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/861b773a93fb48088617bcbb6b8177ec, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b453a1c1783249568f60a0ca2a56ef16] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=48.5 K 2024-11-18T20:10:45,268 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e2c7ac4047450db5f82f76e1650872, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960640909 2024-11-18T20:10:45,269 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d682c1138e4af18db9fbc84afce282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731960642099 2024-11-18T20:10:45,269 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into 1d0b3c2989164a8b8f28349a39bd1584(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:45,269 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 861b773a93fb48088617bcbb6b8177ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960642285 2024-11-18T20:10:45,269 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:45,269 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=12, startTime=1731960645165; duration=0sec 2024-11-18T20:10:45,269 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:45,269 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:45,269 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b453a1c1783249568f60a0ca2a56ef16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960643476 2024-11-18T20:10:45,277 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:45,278 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d0e2189e9b74497d9956181dfabee5c5 is 50, key is test_row_0/B:col10/1731960643486/Put/seqid=0 2024-11-18T20:10:45,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742341_1517 (size=13017) 2024-11-18T20:10:45,288 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/d0e2189e9b74497d9956181dfabee5c5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0e2189e9b74497d9956181dfabee5c5 2024-11-18T20:10:45,293 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into d0e2189e9b74497d9956181dfabee5c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:45,293 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:45,293 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=12, startTime=1731960645166; duration=0sec 2024-11-18T20:10:45,293 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:45,293 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:45,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:10:45,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/906cedbebf2846f688f66ed0e4abe906 is 50, key is test_row_0/A:col10/1731960644670/Put/seqid=0 2024-11-18T20:10:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742342_1518 (size=12301) 2024-11-18T20:10:45,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/906cedbebf2846f688f66ed0e4abe906 2024-11-18T20:10:45,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/2ac2bcffbed74a64824218b710bdd24a is 50, key is test_row_0/B:col10/1731960644670/Put/seqid=0 2024-11-18T20:10:45,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742343_1519 (size=12301) 2024-11-18T20:10:45,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960705340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960705340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960705340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960705342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960705349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960705452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960705452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960705452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960705452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960705461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960705663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960705663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960705663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960705663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960705672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:45,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/2ac2bcffbed74a64824218b710bdd24a 2024-11-18T20:10:45,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/981c3c63fb5747b7a22ce46bdc2a71ca is 50, key is test_row_0/C:col10/1731960644670/Put/seqid=0 2024-11-18T20:10:45,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742344_1520 (size=12301) 2024-11-18T20:10:45,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/981c3c63fb5747b7a22ce46bdc2a71ca 2024-11-18T20:10:45,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/906cedbebf2846f688f66ed0e4abe906 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/906cedbebf2846f688f66ed0e4abe906 2024-11-18T20:10:45,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/906cedbebf2846f688f66ed0e4abe906, entries=150, sequenceid=332, filesize=12.0 K 2024-11-18T20:10:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/2ac2bcffbed74a64824218b710bdd24a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ac2bcffbed74a64824218b710bdd24a 2024-11-18T20:10:45,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ac2bcffbed74a64824218b710bdd24a, entries=150, sequenceid=332, filesize=12.0 K 2024-11-18T20:10:45,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/981c3c63fb5747b7a22ce46bdc2a71ca as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/981c3c63fb5747b7a22ce46bdc2a71ca 2024-11-18T20:10:45,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/981c3c63fb5747b7a22ce46bdc2a71ca, entries=150, sequenceid=332, filesize=12.0 K 2024-11-18T20:10:45,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 471b1d433f657c986c9e035c4a593a36 in 459ms, sequenceid=332, compaction requested=false 2024-11-18T20:10:45,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:45,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:10:45,976 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:45,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-18T20:10:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:45,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/72f94ee13dca4ad29c283822244d41fb is 50, key is test_row_0/A:col10/1731960645339/Put/seqid=0 2024-11-18T20:10:45,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742345_1521 (size=14741) 2024-11-18T20:10:45,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/72f94ee13dca4ad29c283822244d41fb 2024-11-18T20:10:46,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/5d622acd5d3d4950b66a1e96a0e2f065 is 50, key is test_row_0/B:col10/1731960645339/Put/seqid=0 2024-11-18T20:10:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960706016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960706020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960706025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960706022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960706028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742346_1522 (size=12301) 2024-11-18T20:10:46,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960706142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960706142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960706147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960706147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960706154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960706356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960706366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960706376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:10:46,402 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:46,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-18T20:10:46,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:46,417 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:46,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:10:46,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/5d622acd5d3d4950b66a1e96a0e2f065 2024-11-18T20:10:46,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/e2449441bdc44358ab1a428976544d45 is 50, key is test_row_0/C:col10/1731960645339/Put/seqid=0 2024-11-18T20:10:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:10:46,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742347_1523 (size=12301) 2024-11-18T20:10:46,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:46,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960706672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960706673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960706673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960706682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960706692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:10:46,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:46,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:46,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:46,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/e2449441bdc44358ab1a428976544d45 2024-11-18T20:10:46,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/72f94ee13dca4ad29c283822244d41fb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/72f94ee13dca4ad29c283822244d41fb 2024-11-18T20:10:47,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/72f94ee13dca4ad29c283822244d41fb, entries=200, sequenceid=351, filesize=14.4 K 2024-11-18T20:10:47,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/5d622acd5d3d4950b66a1e96a0e2f065 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/5d622acd5d3d4950b66a1e96a0e2f065 2024-11-18T20:10:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:10:47,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/5d622acd5d3d4950b66a1e96a0e2f065, entries=150, sequenceid=351, filesize=12.0 K 2024-11-18T20:10:47,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/e2449441bdc44358ab1a428976544d45 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/e2449441bdc44358ab1a428976544d45 2024-11-18T20:10:47,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/e2449441bdc44358ab1a428976544d45, entries=150, sequenceid=351, filesize=12.0 K 2024-11-18T20:10:47,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 471b1d433f657c986c9e035c4a593a36 in 1165ms, sequenceid=351, compaction requested=true 2024-11-18T20:10:47,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:47,142 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:47,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:47,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:47,142 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:47,148 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:47,148 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/A is initiating minor compaction (all files) 2024-11-18T20:10:47,148 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/A in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,148 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d0b3c2989164a8b8f28349a39bd1584, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/906cedbebf2846f688f66ed0e4abe906, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/72f94ee13dca4ad29c283822244d41fb] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=39.1 K 2024-11-18T20:10:47,148 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:47,149 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/B is initiating minor compaction (all files) 2024-11-18T20:10:47,149 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/B in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,149 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0e2189e9b74497d9956181dfabee5c5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ac2bcffbed74a64824218b710bdd24a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/5d622acd5d3d4950b66a1e96a0e2f065] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=36.7 K 2024-11-18T20:10:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 471b1d433f657c986c9e035c4a593a36:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:47,153 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e2189e9b74497d9956181dfabee5c5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960643476 2024-11-18T20:10:47,153 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0b3c2989164a8b8f28349a39bd1584, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960643476 2024-11-18T20:10:47,153 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac2bcffbed74a64824218b710bdd24a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731960644670 2024-11-18T20:10:47,153 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906cedbebf2846f688f66ed0e4abe906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731960644670 2024-11-18T20:10:47,154 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d622acd5d3d4950b66a1e96a0e2f065, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731960645339 2024-11-18T20:10:47,154 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72f94ee13dca4ad29c283822244d41fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731960645339 2024-11-18T20:10:47,199 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:47,200 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/054ee12e31cc4f07b8e29781c34b51dc is 50, key is test_row_0/B:col10/1731960645339/Put/seqid=0 2024-11-18T20:10:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:47,209 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:47,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:10:47,213 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:47,214 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/23948ed3c53a4e61b2b3d70d86e68d97 is 50, key is test_row_0/A:col10/1731960645339/Put/seqid=0 2024-11-18T20:10:47,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742349_1525 (size=13119) 2024-11-18T20:10:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742348_1524 (size=13119) 2024-11-18T20:10:47,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/3cbeb3846e1b4c04a548e9136e68cc39 is 50, key is test_row_0/A:col10/1731960647201/Put/seqid=0 2024-11-18T20:10:47,271 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/23948ed3c53a4e61b2b3d70d86e68d97 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/23948ed3c53a4e61b2b3d70d86e68d97 2024-11-18T20:10:47,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742350_1526 (size=14741) 2024-11-18T20:10:47,303 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/A of 471b1d433f657c986c9e035c4a593a36 into 23948ed3c53a4e61b2b3d70d86e68d97(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:47,303 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:47,303 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/A, priority=13, startTime=1731960647142; duration=0sec 2024-11-18T20:10:47,304 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:47,304 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:A 2024-11-18T20:10:47,304 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960707284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960707286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960707288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960707290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,312 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:47,312 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): 471b1d433f657c986c9e035c4a593a36/C is initiating minor compaction (all files) 2024-11-18T20:10:47,312 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 471b1d433f657c986c9e035c4a593a36/C in TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,312 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/4056abc5a51c45d4a1291858bddce1e4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/981c3c63fb5747b7a22ce46bdc2a71ca, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/e2449441bdc44358ab1a428976544d45] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp, totalSize=36.7 K 2024-11-18T20:10:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,312 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4056abc5a51c45d4a1291858bddce1e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960643476 2024-11-18T20:10:47,312 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981c3c63fb5747b7a22ce46bdc2a71ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731960644670 2024-11-18T20:10:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,319 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2449441bdc44358ab1a428976544d45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731960645339 2024-11-18T20:10:47,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960707311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,341 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 471b1d433f657c986c9e035c4a593a36#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:47,341 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/6de27eab3776419d916d4d8eb9d088a0 is 50, key is test_row_0/C:col10/1731960645339/Put/seqid=0 2024-11-18T20:10:47,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742351_1527 (size=13119) 2024-11-18T20:10:47,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:47,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:47,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960707414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960707414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,432 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-18T20:10:47,432 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:47,432 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,432 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:47,437 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-11-18T20:10:47,437 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:47,437 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,437 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960707430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960707431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,441 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-18T20:10:47,441 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:47,441 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,442 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960707443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,445 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-11-18T20:10:47,445 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:47,445 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,445 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:47,450 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-18T20:10:47,450 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:47,450 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,450 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:47,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:47,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:47,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:10:47,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960707632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960707632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960707642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,651 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/054ee12e31cc4f07b8e29781c34b51dc as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/054ee12e31cc4f07b8e29781c34b51dc 2024-11-18T20:10:47,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960707652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960707652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,655 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/B of 471b1d433f657c986c9e035c4a593a36 into 054ee12e31cc4f07b8e29781c34b51dc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:47,655 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:47,655 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/B, priority=13, startTime=1731960647142; duration=0sec 2024-11-18T20:10:47,655 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:47,655 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:B 2024-11-18T20:10:47,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/3cbeb3846e1b4c04a548e9136e68cc39 2024-11-18T20:10:47,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/f51853adb1f843629a5e506002383cb6 is 50, key is test_row_0/B:col10/1731960647201/Put/seqid=0 2024-11-18T20:10:47,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742352_1528 (size=12301) 2024-11-18T20:10:47,797 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/6de27eab3776419d916d4d8eb9d088a0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/6de27eab3776419d916d4d8eb9d088a0 2024-11-18T20:10:47,802 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 471b1d433f657c986c9e035c4a593a36/C of 471b1d433f657c986c9e035c4a593a36 into 6de27eab3776419d916d4d8eb9d088a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:47,802 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:47,802 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., storeName=471b1d433f657c986c9e035c4a593a36/C, priority=13, startTime=1731960647149; duration=0sec 2024-11-18T20:10:47,802 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:47,802 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 471b1d433f657c986c9e035c4a593a36:C 2024-11-18T20:10:47,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:47,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:47,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45906 deadline: 1731960707942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45976 deadline: 1731960707942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45962 deadline: 1731960707952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45934 deadline: 1731960707962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45928 deadline: 1731960707962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., hostname=d8f95bb18b58,42447,1731960497538, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:10:48,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:48,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. as already flushing 2024-11-18T20:10:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:48,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/f51853adb1f843629a5e506002383cb6 2024-11-18T20:10:48,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a is 50, key is test_row_0/C:col10/1731960647201/Put/seqid=0 2024-11-18T20:10:48,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742353_1529 (size=12301) 2024-11-18T20:10:48,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a 2024-11-18T20:10:48,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/3cbeb3846e1b4c04a548e9136e68cc39 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/3cbeb3846e1b4c04a548e9136e68cc39 2024-11-18T20:10:48,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/3cbeb3846e1b4c04a548e9136e68cc39, entries=200, sequenceid=369, filesize=14.4 K 2024-11-18T20:10:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/f51853adb1f843629a5e506002383cb6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/f51853adb1f843629a5e506002383cb6 2024-11-18T20:10:48,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/f51853adb1f843629a5e506002383cb6, entries=150, sequenceid=369, filesize=12.0 K 2024-11-18T20:10:48,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a 2024-11-18T20:10:48,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-18T20:10:48,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 471b1d433f657c986c9e035c4a593a36 in 985ms, sequenceid=369, compaction requested=false 2024-11-18T20:10:48,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:10:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,194 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:10:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/efbcd47d831547c88101e2756b1e249e is 50, key is test_row_0/A:col10/1731960647247/Put/seqid=0 2024-11-18T20:10:48,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742354_1530 (size=12301) 2024-11-18T20:10:48,225 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/efbcd47d831547c88101e2756b1e249e 2024-11-18T20:10:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/883b37e44b2d418e940e11f351840c40 is 50, key is test_row_0/B:col10/1731960647247/Put/seqid=0 2024-11-18T20:10:48,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742355_1531 (size=12301) 2024-11-18T20:10:48,277 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/883b37e44b2d418e940e11f351840c40 2024-11-18T20:10:48,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/07887620f5684596a72f3f13ee043a2a is 50, key is test_row_0/C:col10/1731960647247/Put/seqid=0 2024-11-18T20:10:48,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742356_1532 (size=12301) 2024-11-18T20:10:48,326 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/07887620f5684596a72f3f13ee043a2a 2024-11-18T20:10:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/efbcd47d831547c88101e2756b1e249e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/efbcd47d831547c88101e2756b1e249e 2024-11-18T20:10:48,368 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/efbcd47d831547c88101e2756b1e249e, entries=150, sequenceid=390, filesize=12.0 K 2024-11-18T20:10:48,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/883b37e44b2d418e940e11f351840c40 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/883b37e44b2d418e940e11f351840c40 2024-11-18T20:10:48,377 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/883b37e44b2d418e940e11f351840c40, entries=150, sequenceid=390, filesize=12.0 K 2024-11-18T20:10:48,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/07887620f5684596a72f3f13ee043a2a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/07887620f5684596a72f3f13ee043a2a 2024-11-18T20:10:48,382 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/07887620f5684596a72f3f13ee043a2a, entries=150, sequenceid=390, filesize=12.0 K 2024-11-18T20:10:48,383 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 471b1d433f657c986c9e035c4a593a36 in 189ms, sequenceid=390, compaction requested=true 2024-11-18T20:10:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 471b1d433f657c986c9e035c4a593a36: 2024-11-18T20:10:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-18T20:10:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-18T20:10:48,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-18T20:10:48,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9680 sec 2024-11-18T20:10:48,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.9840 sec 2024-11-18T20:10:48,454 INFO [Thread-2027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2027. 2024-11-18T20:10:48,454 DEBUG [Thread-2027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:48,454 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,454 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:48,454 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-11-18T20:10:48,454 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:48,454 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,454 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:48,464 INFO [Thread-2023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2023. 2024-11-18T20:10:48,464 DEBUG [Thread-2023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:48,464 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,464 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:48,473 INFO [Thread-2025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2025. 2024-11-18T20:10:48,473 DEBUG [Thread-2025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:48,473 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,473 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-11-18T20:10:48,474 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:10:48,474 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,474 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,474 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:48,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:10:48,582 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1003 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2992 rows 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2970 rows 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 989 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2956 rows 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1001 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3003 rows 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-11-18T20:10:48,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2982 rows 2024-11-18T20:10:48,583 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:10:48,583 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:10:48,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:48,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:10:48,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:10:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:10:48,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960648589"}]},"ts":"1731960648589"} 2024-11-18T20:10:48,590 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:10:48,590 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:10:48,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:10:48,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, UNASSIGN}] 2024-11-18T20:10:48,595 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, UNASSIGN 2024-11-18T20:10:48,596 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=471b1d433f657c986c9e035c4a593a36, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:48,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, UNASSIGN because future has completed 2024-11-18T20:10:48,598 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:10:48,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:10:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:10:48,752 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:48,752 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:10:48,752 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 471b1d433f657c986c9e035c4a593a36, disabling compactions & flushes 2024-11-18T20:10:48,752 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,752 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,752 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. after waiting 0 ms 2024-11-18T20:10:48,752 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:48,752 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 471b1d433f657c986c9e035c4a593a36 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:10:48,756 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/9468f820451541968cb2f0e871743bea is 50, key is test_row_0/A:col10/1731960648452/Put/seqid=0 2024-11-18T20:10:48,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742357_1533 (size=9857) 2024-11-18T20:10:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:10:49,177 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/9468f820451541968cb2f0e871743bea 2024-11-18T20:10:49,184 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/2ec48013f0a7447b92ab3f4b9cf78cca is 50, key is test_row_0/B:col10/1731960648452/Put/seqid=0 2024-11-18T20:10:49,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742358_1534 (size=9857) 2024-11-18T20:10:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:10:49,219 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/2ec48013f0a7447b92ab3f4b9cf78cca 2024-11-18T20:10:49,229 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a052dc2a66e64f4b9456fe6f4cdb3dcd is 50, key is test_row_0/C:col10/1731960648452/Put/seqid=0 2024-11-18T20:10:49,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742359_1535 (size=9857) 2024-11-18T20:10:49,637 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a052dc2a66e64f4b9456fe6f4cdb3dcd 2024-11-18T20:10:49,641 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/A/9468f820451541968cb2f0e871743bea as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/9468f820451541968cb2f0e871743bea 2024-11-18T20:10:49,645 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/9468f820451541968cb2f0e871743bea, entries=100, sequenceid=398, filesize=9.6 K 2024-11-18T20:10:49,646 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/B/2ec48013f0a7447b92ab3f4b9cf78cca as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ec48013f0a7447b92ab3f4b9cf78cca 2024-11-18T20:10:49,651 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ec48013f0a7447b92ab3f4b9cf78cca, entries=100, sequenceid=398, filesize=9.6 K 2024-11-18T20:10:49,651 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/.tmp/C/a052dc2a66e64f4b9456fe6f4cdb3dcd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a052dc2a66e64f4b9456fe6f4cdb3dcd 2024-11-18T20:10:49,655 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a052dc2a66e64f4b9456fe6f4cdb3dcd, entries=100, sequenceid=398, filesize=9.6 K 2024-11-18T20:10:49,656 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 471b1d433f657c986c9e035c4a593a36 in 904ms, sequenceid=398, compaction requested=true 2024-11-18T20:10:49,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/4a00b8a71d4b49cd9a4d3d847e68d111, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/44171d59fedf497f9d20331b964ab625, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ec272de7c6ee4446be4998a9dcc7e5b2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7de6d575f4fa4e0d88572af846d5e5c3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7b2d3b3dc37340278564aedb0969eaa2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b700df98ff9844ceb0822eda3f2fd86a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/961086cd99ea471b819e527fd9187f95, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b59385e78b9e4737bf9c81b5d68ef8e4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b100f90ee210417eae2c0073191db16b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/c7050d61f7e643ed8d605d67fcaacbe1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/fc6b8fe712f2410fb363d94398f45fa3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/61ecd95f04d548bda3d073c7337ce86d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/8b8601fbf10d4cae8646b2e9b07c2703, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ba12d14499a340e082e3bb376fcbe1c5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1cda25c544b4424f8134c1ad5d27d81c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/2a6ed24c5df543478d4786a426cb35b7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/d0185f0dc00f47fe8554118f6c7c6411, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/02304e647e2a46d18bc4508dd0dbcc0f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/0fb4dfd25ca540ee8f608c670e596809, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/43cee4172e9a4164995024d2e6d3d384, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d92dcd46caa4c6da5aea82ce483dfa2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d0b3c2989164a8b8f28349a39bd1584, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/906cedbebf2846f688f66ed0e4abe906, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/72f94ee13dca4ad29c283822244d41fb] to archive 2024-11-18T20:10:49,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:10:49,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/4a00b8a71d4b49cd9a4d3d847e68d111 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/4a00b8a71d4b49cd9a4d3d847e68d111 2024-11-18T20:10:49,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/44171d59fedf497f9d20331b964ab625 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/44171d59fedf497f9d20331b964ab625 2024-11-18T20:10:49,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ec272de7c6ee4446be4998a9dcc7e5b2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ec272de7c6ee4446be4998a9dcc7e5b2 2024-11-18T20:10:49,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7de6d575f4fa4e0d88572af846d5e5c3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7de6d575f4fa4e0d88572af846d5e5c3 2024-11-18T20:10:49,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7b2d3b3dc37340278564aedb0969eaa2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/7b2d3b3dc37340278564aedb0969eaa2 2024-11-18T20:10:49,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b700df98ff9844ceb0822eda3f2fd86a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b700df98ff9844ceb0822eda3f2fd86a 2024-11-18T20:10:49,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/961086cd99ea471b819e527fd9187f95 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/961086cd99ea471b819e527fd9187f95 2024-11-18T20:10:49,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b59385e78b9e4737bf9c81b5d68ef8e4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b59385e78b9e4737bf9c81b5d68ef8e4 2024-11-18T20:10:49,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b100f90ee210417eae2c0073191db16b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/b100f90ee210417eae2c0073191db16b 2024-11-18T20:10:49,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/c7050d61f7e643ed8d605d67fcaacbe1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/c7050d61f7e643ed8d605d67fcaacbe1 2024-11-18T20:10:49,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/fc6b8fe712f2410fb363d94398f45fa3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/fc6b8fe712f2410fb363d94398f45fa3 2024-11-18T20:10:49,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/61ecd95f04d548bda3d073c7337ce86d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/61ecd95f04d548bda3d073c7337ce86d 2024-11-18T20:10:49,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/8b8601fbf10d4cae8646b2e9b07c2703 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/8b8601fbf10d4cae8646b2e9b07c2703 2024-11-18T20:10:49,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ba12d14499a340e082e3bb376fcbe1c5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/ba12d14499a340e082e3bb376fcbe1c5 2024-11-18T20:10:49,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1cda25c544b4424f8134c1ad5d27d81c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1cda25c544b4424f8134c1ad5d27d81c 2024-11-18T20:10:49,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/2a6ed24c5df543478d4786a426cb35b7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/2a6ed24c5df543478d4786a426cb35b7 2024-11-18T20:10:49,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/d0185f0dc00f47fe8554118f6c7c6411 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/d0185f0dc00f47fe8554118f6c7c6411 2024-11-18T20:10:49,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/02304e647e2a46d18bc4508dd0dbcc0f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/02304e647e2a46d18bc4508dd0dbcc0f 2024-11-18T20:10:49,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/0fb4dfd25ca540ee8f608c670e596809 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/0fb4dfd25ca540ee8f608c670e596809 2024-11-18T20:10:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:10:49,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/43cee4172e9a4164995024d2e6d3d384 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/43cee4172e9a4164995024d2e6d3d384 2024-11-18T20:10:49,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d92dcd46caa4c6da5aea82ce483dfa2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d92dcd46caa4c6da5aea82ce483dfa2 2024-11-18T20:10:49,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d0b3c2989164a8b8f28349a39bd1584 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/1d0b3c2989164a8b8f28349a39bd1584 2024-11-18T20:10:49,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/906cedbebf2846f688f66ed0e4abe906 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/906cedbebf2846f688f66ed0e4abe906 2024-11-18T20:10:49,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/72f94ee13dca4ad29c283822244d41fb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/72f94ee13dca4ad29c283822244d41fb 2024-11-18T20:10:49,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/a2b133365c524d869f0bb48903d61d56, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d431c6e580cb4980adf6b73578e8da18, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/be667ab04140400c86099c4cdc856d42, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/c89b26afa2b346b89480f555100af068, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/e3c42a3a0dce486598df04e5ad42d53c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/cab4af8f8be4402e94ac7b507852af86, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/0f69b4dd95f1461d90bd5cc147cdde33, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6f40e7bbeb344bcb9b7a6984852455ee, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/edd59e8768364cd18ea5c7ca9f252125, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/878b55cb22dc4963a3ad82b746da59b4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/dd161620146c4a71a2578e9a420d61ed, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6a1f40809e644819825dded429806369, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0bccba6b0024258b2b1b483f46fcd5e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b019675916a94b4cbf4524392af0463a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b209937d2305421e9c2c317f8f9a0707, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/04e2c7ac4047450db5f82f76e1650872, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/4c5a7c2dfa4d4ed78fc6562302b2136d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/52d682c1138e4af18db9fbc84afce282, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/861b773a93fb48088617bcbb6b8177ec, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0e2189e9b74497d9956181dfabee5c5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b453a1c1783249568f60a0ca2a56ef16, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ac2bcffbed74a64824218b710bdd24a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/5d622acd5d3d4950b66a1e96a0e2f065] to archive 2024-11-18T20:10:49,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:10:49,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/a2b133365c524d869f0bb48903d61d56 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/a2b133365c524d869f0bb48903d61d56 2024-11-18T20:10:49,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d431c6e580cb4980adf6b73578e8da18 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d431c6e580cb4980adf6b73578e8da18 2024-11-18T20:10:49,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/be667ab04140400c86099c4cdc856d42 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/be667ab04140400c86099c4cdc856d42 2024-11-18T20:10:49,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/c89b26afa2b346b89480f555100af068 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/c89b26afa2b346b89480f555100af068 2024-11-18T20:10:49,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/e3c42a3a0dce486598df04e5ad42d53c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/e3c42a3a0dce486598df04e5ad42d53c 2024-11-18T20:10:49,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/cab4af8f8be4402e94ac7b507852af86 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/cab4af8f8be4402e94ac7b507852af86 2024-11-18T20:10:49,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/0f69b4dd95f1461d90bd5cc147cdde33 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/0f69b4dd95f1461d90bd5cc147cdde33 2024-11-18T20:10:49,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6f40e7bbeb344bcb9b7a6984852455ee to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6f40e7bbeb344bcb9b7a6984852455ee 2024-11-18T20:10:49,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/90d3c66ec5464f2e9d2d4e7efb7b9ea0 2024-11-18T20:10:49,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/edd59e8768364cd18ea5c7ca9f252125 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/edd59e8768364cd18ea5c7ca9f252125 2024-11-18T20:10:49,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/878b55cb22dc4963a3ad82b746da59b4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/878b55cb22dc4963a3ad82b746da59b4 2024-11-18T20:10:49,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/dd161620146c4a71a2578e9a420d61ed to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/dd161620146c4a71a2578e9a420d61ed 2024-11-18T20:10:49,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6a1f40809e644819825dded429806369 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/6a1f40809e644819825dded429806369 2024-11-18T20:10:49,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0bccba6b0024258b2b1b483f46fcd5e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0bccba6b0024258b2b1b483f46fcd5e 2024-11-18T20:10:49,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b019675916a94b4cbf4524392af0463a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b019675916a94b4cbf4524392af0463a 2024-11-18T20:10:49,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b209937d2305421e9c2c317f8f9a0707 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b209937d2305421e9c2c317f8f9a0707 2024-11-18T20:10:49,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/04e2c7ac4047450db5f82f76e1650872 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/04e2c7ac4047450db5f82f76e1650872 2024-11-18T20:10:49,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/4c5a7c2dfa4d4ed78fc6562302b2136d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/4c5a7c2dfa4d4ed78fc6562302b2136d 2024-11-18T20:10:49,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/52d682c1138e4af18db9fbc84afce282 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/52d682c1138e4af18db9fbc84afce282 2024-11-18T20:10:49,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/861b773a93fb48088617bcbb6b8177ec to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/861b773a93fb48088617bcbb6b8177ec 2024-11-18T20:10:49,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0e2189e9b74497d9956181dfabee5c5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/d0e2189e9b74497d9956181dfabee5c5 2024-11-18T20:10:49,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b453a1c1783249568f60a0ca2a56ef16 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/b453a1c1783249568f60a0ca2a56ef16 2024-11-18T20:10:49,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ac2bcffbed74a64824218b710bdd24a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ac2bcffbed74a64824218b710bdd24a 2024-11-18T20:10:49,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/5d622acd5d3d4950b66a1e96a0e2f065 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/5d622acd5d3d4950b66a1e96a0e2f065 2024-11-18T20:10:50,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/33c23f393cd04ffb9f0ffe189511af89, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/8b037ee2e23b4d168af7727b7d6d80a2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3477e0287b8a4995b21cfa8e91583cd3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/db586b7f9cf74423a2e7394035519a9f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a4cbbba40a4e4a999b13c0b24028bd2b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/d22c20d6e9c940c9a6c88533900cf18b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/de1c1d6082304e969b9791db9c7adae8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/baa554cb5a244214b3f6240a8604ef1c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/27bbea9977bc4d2b938281f169821db3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/506d09b836794784864cfee4ce62b3c4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/cb0ee776e7ff4591875d13dbfaddd812, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/ced90f472f624eeb99cb361e6cb057d7, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/843e043dfcf844ca92389a8bc51bf0bf, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/18b4377ce32046f0a5790d7d0a0769ba, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/9c5556f72cb049acb62029d98b21d326, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/c59aed260dee44e9ba2452806c9946fa, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/51954da6d4314c7c8d70754d123f6451, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a35d686054b84b3fb8bf8c3cfdd4df5c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/63699205ca204a7c8701a6fc2248cfba, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3c3445b972f142f98aa3cbe82a2101c4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/4056abc5a51c45d4a1291858bddce1e4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/947c445c67214619bb37652dc267f46d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/981c3c63fb5747b7a22ce46bdc2a71ca, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/e2449441bdc44358ab1a428976544d45] to archive 2024-11-18T20:10:50,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:10:50,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/33c23f393cd04ffb9f0ffe189511af89 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/33c23f393cd04ffb9f0ffe189511af89 2024-11-18T20:10:50,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/8b037ee2e23b4d168af7727b7d6d80a2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/8b037ee2e23b4d168af7727b7d6d80a2 2024-11-18T20:10:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3477e0287b8a4995b21cfa8e91583cd3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3477e0287b8a4995b21cfa8e91583cd3 2024-11-18T20:10:50,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/db586b7f9cf74423a2e7394035519a9f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/db586b7f9cf74423a2e7394035519a9f 2024-11-18T20:10:50,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a4cbbba40a4e4a999b13c0b24028bd2b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a4cbbba40a4e4a999b13c0b24028bd2b 2024-11-18T20:10:50,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/d22c20d6e9c940c9a6c88533900cf18b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/d22c20d6e9c940c9a6c88533900cf18b 2024-11-18T20:10:50,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/de1c1d6082304e969b9791db9c7adae8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/de1c1d6082304e969b9791db9c7adae8 2024-11-18T20:10:50,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/baa554cb5a244214b3f6240a8604ef1c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/baa554cb5a244214b3f6240a8604ef1c 2024-11-18T20:10:50,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/27bbea9977bc4d2b938281f169821db3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/27bbea9977bc4d2b938281f169821db3 2024-11-18T20:10:50,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/506d09b836794784864cfee4ce62b3c4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/506d09b836794784864cfee4ce62b3c4 2024-11-18T20:10:50,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/cb0ee776e7ff4591875d13dbfaddd812 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/cb0ee776e7ff4591875d13dbfaddd812 2024-11-18T20:10:50,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/ced90f472f624eeb99cb361e6cb057d7 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/ced90f472f624eeb99cb361e6cb057d7 2024-11-18T20:10:50,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/843e043dfcf844ca92389a8bc51bf0bf to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/843e043dfcf844ca92389a8bc51bf0bf 2024-11-18T20:10:50,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/18b4377ce32046f0a5790d7d0a0769ba to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/18b4377ce32046f0a5790d7d0a0769ba 2024-11-18T20:10:50,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/9c5556f72cb049acb62029d98b21d326 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/9c5556f72cb049acb62029d98b21d326 2024-11-18T20:10:50,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/c59aed260dee44e9ba2452806c9946fa to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/c59aed260dee44e9ba2452806c9946fa 2024-11-18T20:10:50,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/51954da6d4314c7c8d70754d123f6451 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/51954da6d4314c7c8d70754d123f6451 2024-11-18T20:10:50,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a35d686054b84b3fb8bf8c3cfdd4df5c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a35d686054b84b3fb8bf8c3cfdd4df5c 2024-11-18T20:10:50,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/63699205ca204a7c8701a6fc2248cfba to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/63699205ca204a7c8701a6fc2248cfba 2024-11-18T20:10:50,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3c3445b972f142f98aa3cbe82a2101c4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/3c3445b972f142f98aa3cbe82a2101c4 2024-11-18T20:10:50,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/4056abc5a51c45d4a1291858bddce1e4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/4056abc5a51c45d4a1291858bddce1e4 2024-11-18T20:10:50,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/947c445c67214619bb37652dc267f46d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/947c445c67214619bb37652dc267f46d 2024-11-18T20:10:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/981c3c63fb5747b7a22ce46bdc2a71ca to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/981c3c63fb5747b7a22ce46bdc2a71ca 2024-11-18T20:10:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/e2449441bdc44358ab1a428976544d45 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/e2449441bdc44358ab1a428976544d45 2024-11-18T20:10:50,351 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=1 2024-11-18T20:10:50,356 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. 2024-11-18T20:10:50,356 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 471b1d433f657c986c9e035c4a593a36: Waiting for close lock at 1731960648752Running coprocessor pre-close hooks at 1731960648752Disabling compacts and flushes for region at 1731960648752Disabling writes for close at 1731960648752Obtaining lock to block concurrent updates at 1731960648752Preparing flush snapshotting stores in 471b1d433f657c986c9e035c4a593a36 at 1731960648752Finished memstore snapshotting TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960648753 (+1 ms)Flushing stores of TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36. at 1731960648753Flushing 471b1d433f657c986c9e035c4a593a36/A: creating writer at 1731960648753Flushing 471b1d433f657c986c9e035c4a593a36/A: appending metadata at 1731960648756 (+3 ms)Flushing 471b1d433f657c986c9e035c4a593a36/A: closing flushed file at 1731960648756Flushing 471b1d433f657c986c9e035c4a593a36/B: creating writer at 1731960649181 (+425 ms)Flushing 471b1d433f657c986c9e035c4a593a36/B: appending metadata at 1731960649183 (+2 ms)Flushing 471b1d433f657c986c9e035c4a593a36/B: closing flushed file at 1731960649183Flushing 471b1d433f657c986c9e035c4a593a36/C: creating writer at 1731960649226 (+43 ms)Flushing 471b1d433f657c986c9e035c4a593a36/C: appending metadata at 1731960649228 (+2 ms)Flushing 471b1d433f657c986c9e035c4a593a36/C: closing flushed file at 1731960649228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@561126c8: reopening flushed file at 1731960649641 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3351002b: reopening flushed file at 1731960649645 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a5d9e6: reopening flushed file at 1731960649651 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 471b1d433f657c986c9e035c4a593a36 in 904ms, sequenceid=398, compaction requested=true at 1731960649656 (+5 ms)Writing region close event to WAL at 1731960650318 (+662 ms)Running coprocessor post-close hooks at 1731960650356 (+38 ms)Closed at 1731960650356 2024-11-18T20:10:50,365 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:50,370 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=471b1d433f657c986c9e035c4a593a36, regionState=CLOSED 2024-11-18T20:10:50,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:10:50,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-18T20:10:50,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 471b1d433f657c986c9e035c4a593a36, server=d8f95bb18b58,42447,1731960497538 in 1.7860 sec 2024-11-18T20:10:50,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-18T20:10:50,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=471b1d433f657c986c9e035c4a593a36, UNASSIGN in 1.7920 sec 2024-11-18T20:10:50,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-18T20:10:50,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7960 sec 2024-11-18T20:10:50,391 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960650391"}]},"ts":"1731960650391"} 2024-11-18T20:10:50,393 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:10:50,393 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:10:50,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8090 sec 2024-11-18T20:10:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:10:50,733 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:50,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:10:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,735 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:10:50,736 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,738 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:50,740 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/recovered.edits] 2024-11-18T20:10:50,742 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/23948ed3c53a4e61b2b3d70d86e68d97 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/23948ed3c53a4e61b2b3d70d86e68d97 2024-11-18T20:10:50,744 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/3cbeb3846e1b4c04a548e9136e68cc39 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/3cbeb3846e1b4c04a548e9136e68cc39 2024-11-18T20:10:50,745 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/9468f820451541968cb2f0e871743bea to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/9468f820451541968cb2f0e871743bea 2024-11-18T20:10:50,746 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/efbcd47d831547c88101e2756b1e249e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/A/efbcd47d831547c88101e2756b1e249e 2024-11-18T20:10:50,754 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/054ee12e31cc4f07b8e29781c34b51dc to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/054ee12e31cc4f07b8e29781c34b51dc 2024-11-18T20:10:50,755 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ec48013f0a7447b92ab3f4b9cf78cca to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/2ec48013f0a7447b92ab3f4b9cf78cca 2024-11-18T20:10:50,756 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/883b37e44b2d418e940e11f351840c40 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/883b37e44b2d418e940e11f351840c40 2024-11-18T20:10:50,757 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/f51853adb1f843629a5e506002383cb6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/B/f51853adb1f843629a5e506002383cb6 2024-11-18T20:10:50,760 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/07887620f5684596a72f3f13ee043a2a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/07887620f5684596a72f3f13ee043a2a 2024-11-18T20:10:50,761 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/2f92fb08750e4f6b8ebb0ccc9ab22a6a 2024-11-18T20:10:50,763 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/6de27eab3776419d916d4d8eb9d088a0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/6de27eab3776419d916d4d8eb9d088a0 2024-11-18T20:10:50,764 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a052dc2a66e64f4b9456fe6f4cdb3dcd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/C/a052dc2a66e64f4b9456fe6f4cdb3dcd 2024-11-18T20:10:50,767 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/recovered.edits/401.seqid to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36/recovered.edits/401.seqid 2024-11-18T20:10:50,768 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/471b1d433f657c986c9e035c4a593a36 2024-11-18T20:10:50,768 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:10:50,771 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,773 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:10:50,775 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:10:50,777 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,777 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:10:50,777 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960650777"}]},"ts":"9223372036854775807"} 2024-11-18T20:10:50,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:10:50,780 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 471b1d433f657c986c9e035c4a593a36, NAME => 'TestAcidGuarantees,,1731960625116.471b1d433f657c986c9e035c4a593a36.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:10:50,780 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:10:50,780 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960650780"}]},"ts":"9223372036854775807"} 2024-11-18T20:10:50,783 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:10:50,784 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-18T20:10:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:10:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:10:50,846 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:50,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=819 (was 587) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1878 (was 1399) - AvailableMemoryMB LEAK? - 2024-11-18T20:10:50,866 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=819, ProcessCount=11, AvailableMemoryMB=1878 2024-11-18T20:10:50,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:10:50,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:10:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:50,870 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:10:50,870 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:50,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-18T20:10:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T20:10:50,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:10:50,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742360_1536 (size=959) 2024-11-18T20:10:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T20:10:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T20:10:51,289 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c 2024-11-18T20:10:51,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742361_1537 (size=53) 2024-11-18T20:10:51,322 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:10:51,322 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fc81ed96fcb7834182497904e66e6d77, disabling compactions & flushes 2024-11-18T20:10:51,323 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. after waiting 0 ms 2024-11-18T20:10:51,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,323 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,323 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc81ed96fcb7834182497904e66e6d77: Waiting for close lock at 1731960651322Disabling compacts and flushes for region at 1731960651322Disabling writes for close at 1731960651323 (+1 ms)Writing region close event to WAL at 1731960651323Closed at 1731960651323 2024-11-18T20:10:51,324 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:10:51,325 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960651324"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960651324"}]},"ts":"1731960651324"} 2024-11-18T20:10:51,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:10:51,328 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:10:51,328 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960651328"}]},"ts":"1731960651328"} 2024-11-18T20:10:51,329 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:10:51,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, ASSIGN}] 2024-11-18T20:10:51,331 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, ASSIGN 2024-11-18T20:10:51,332 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, ASSIGN; state=OFFLINE, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=false 2024-11-18T20:10:51,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:51,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, ASSIGN because future has completed 2024-11-18T20:10:51,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:10:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T20:10:51,640 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,640 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:10:51,640 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,640 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:10:51,640 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,640 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,642 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,643 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc81ed96fcb7834182497904e66e6d77 columnFamilyName A 2024-11-18T20:10:51,643 DEBUG [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:51,644 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(327): Store=fc81ed96fcb7834182497904e66e6d77/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:51,644 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,646 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc81ed96fcb7834182497904e66e6d77 columnFamilyName B 2024-11-18T20:10:51,646 DEBUG [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:51,646 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(327): Store=fc81ed96fcb7834182497904e66e6d77/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:51,647 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,648 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc81ed96fcb7834182497904e66e6d77 columnFamilyName C 2024-11-18T20:10:51,648 DEBUG [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:51,649 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(327): Store=fc81ed96fcb7834182497904e66e6d77/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:51,649 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,649 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,650 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,651 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,651 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,651 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:10:51,652 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,654 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:10:51,655 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened fc81ed96fcb7834182497904e66e6d77; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68158405, jitterRate=0.015639379620552063}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:10:51,655 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:51,656 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for fc81ed96fcb7834182497904e66e6d77: Running coprocessor pre-open hook at 1731960651641Writing region info on filesystem at 1731960651641Initializing all the Stores at 1731960651641Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960651641Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960651641Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960651642 (+1 ms)Cleaning up temporary data from old regions at 1731960651651 (+9 ms)Running coprocessor post-open hooks at 1731960651655 (+4 ms)Region opened successfully at 1731960651656 (+1 ms) 2024-11-18T20:10:51,657 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., pid=144, masterSystemTime=1731960651636 2024-11-18T20:10:51,658 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,659 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:51,659 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=OPEN, openSeqNum=2, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:51,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:10:51,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-18T20:10:51,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 in 178 msec 2024-11-18T20:10:51,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-18T20:10:51,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, ASSIGN in 334 msec 2024-11-18T20:10:51,668 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:10:51,668 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960651668"}]},"ts":"1731960651668"} 2024-11-18T20:10:51,672 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:10:51,673 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:10:51,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 805 msec 2024-11-18T20:10:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T20:10:52,012 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:52,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:52,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:52,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:52,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6ba830, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:52,018 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3679c04e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:52,019 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:52,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:52,022 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:10:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d8f95bb18b58,42935,1731960496743 2024-11-18T20:10:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@766e47d8 2024-11-18T20:10:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:10:52,024 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:10:52,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:10:52,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:10:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:10:52,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742362_1538 (size=995) 2024-11-18T20:10:52,481 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-18T20:10:52,481 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-18T20:10:52,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:10:52,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, REOPEN/MOVE}] 2024-11-18T20:10:52,489 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, REOPEN/MOVE 2024-11-18T20:10:52,490 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:52,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, REOPEN/MOVE because future has completed 2024-11-18T20:10:52,492 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:10:52,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:10:52,645 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:52,645 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:10:52,645 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing fc81ed96fcb7834182497904e66e6d77, disabling compactions & flushes 2024-11-18T20:10:52,645 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:52,645 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:52,645 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. after waiting 0 ms 2024-11-18T20:10:52,645 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:52,649 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:10:52,650 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:52,650 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for fc81ed96fcb7834182497904e66e6d77: Waiting for close lock at 1731960652645Running coprocessor pre-close hooks at 1731960652645Disabling compacts and flushes for region at 1731960652645Disabling writes for close at 1731960652645Writing region close event to WAL at 1731960652646 (+1 ms)Running coprocessor post-close hooks at 1731960652650 (+4 ms)Closed at 1731960652650 2024-11-18T20:10:52,650 WARN [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: fc81ed96fcb7834182497904e66e6d77 to self. 2024-11-18T20:10:52,652 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:52,652 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=CLOSED 2024-11-18T20:10:52,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:10:52,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-18T20:10:52,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 in 163 msec 2024-11-18T20:10:52,658 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, REOPEN/MOVE; state=CLOSED, location=d8f95bb18b58,42447,1731960497538; forceNewPlan=false, retain=true 2024-11-18T20:10:52,809 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=OPENING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:52,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, REOPEN/MOVE because future has completed 2024-11-18T20:10:52,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:10:52,966 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:52,966 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:10:52,966 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:52,966 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:10:52,966 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:52,966 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:52,967 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:52,976 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc81ed96fcb7834182497904e66e6d77 columnFamilyName A 2024-11-18T20:10:52,992 DEBUG [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:52,999 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(327): Store=fc81ed96fcb7834182497904e66e6d77/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:52,999 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,000 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc81ed96fcb7834182497904e66e6d77 columnFamilyName B 2024-11-18T20:10:53,000 DEBUG [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:53,000 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(327): Store=fc81ed96fcb7834182497904e66e6d77/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:53,001 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,001 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc81ed96fcb7834182497904e66e6d77 columnFamilyName C 2024-11-18T20:10:53,001 DEBUG [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:53,001 INFO [StoreOpener-fc81ed96fcb7834182497904e66e6d77-1 {}] regionserver.HStore(327): Store=fc81ed96fcb7834182497904e66e6d77/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:10:53,002 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,002 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,003 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,004 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,004 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,005 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:10:53,006 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,007 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened fc81ed96fcb7834182497904e66e6d77; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67734420, jitterRate=0.009321510791778564}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:10:53,007 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,008 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for fc81ed96fcb7834182497904e66e6d77: Running coprocessor pre-open hook at 1731960652966Writing region info on filesystem at 1731960652966Initializing all the Stores at 1731960652967 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960652967Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960652967Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960652967Cleaning up temporary data from old regions at 1731960653004 (+37 ms)Running coprocessor post-open hooks at 1731960653007 (+3 ms)Region opened successfully at 1731960653008 (+1 ms) 2024-11-18T20:10:53,009 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., pid=149, masterSystemTime=1731960652963 2024-11-18T20:10:53,010 DEBUG [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,010 INFO [RS_OPEN_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,011 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=OPEN, openSeqNum=5, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:10:53,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-18T20:10:53,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 in 202 msec 2024-11-18T20:10:53,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-18T20:10:53,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, REOPEN/MOVE in 526 msec 2024-11-18T20:10:53,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-18T20:10:53,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-11-18T20:10:53,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 993 msec 2024-11-18T20:10:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-18T20:10:53,023 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:53,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ea1908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1bb28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,030 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a883de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,034 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7ee4ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,041 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c379f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,044 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728fb9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,047 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87a1add, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c42b362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,052 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b30247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,054 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33793f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,060 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e09e6f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,062 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337b5cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a109c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,069 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f98044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,075 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e302d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,077 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289c9761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,080 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f39b738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6781b55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d8f95bb18b58,42935,-1 for getting cluster id 2024-11-18T20:10:53,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:10:53,094 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10a39af9-8ba6-46dd-ada3-b4b0ab3511f2' 2024-11-18T20:10:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:10:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10a39af9-8ba6-46dd-ada3-b4b0ab3511f2" 2024-11-18T20:10:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69544af7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d8f95bb18b58,42935,-1] 2024-11-18T20:10:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:10:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:10:53,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:10:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509a7b2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:10:53,098 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,098 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:53,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,103 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-18T20:10:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,106 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:10:53,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:53,110 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,111 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,111 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:53,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,112 DEBUG [Thread-2406 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,112 DEBUG [Thread-2406 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,116 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,117 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,120 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,122 DEBUG [Thread-2401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:10:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d8f95bb18b58,42447,1731960497538, seqNum=-1] 2024-11-18T20:10:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:10:53,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:10:53,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5] 2024-11-18T20:10:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:53,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:10:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:10:53,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b8d0b371d80341f694d612c3a5990bd2_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960653145/Put/seqid=0 2024-11-18T20:10:53,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960713264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742363_1539 (size=12154) 2024-11-18T20:10:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960713272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960713278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960713282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960713272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960713395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960713396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960713396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960713396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960713396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:10:53,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:53,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:53,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960713601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960713612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960713612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960713612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960713615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:53,726 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b8d0b371d80341f694d612c3a5990bd2_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b8d0b371d80341f694d612c3a5990bd2_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:53,729 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7cd4be5bb0c74e6e8ed5d5b4968557f8, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:53,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7cd4be5bb0c74e6e8ed5d5b4968557f8 is 175, key is test_row_0/A:col10/1731960653145/Put/seqid=0 2024-11-18T20:10:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:10:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742364_1540 (size=30955) 2024-11-18T20:10:53,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:53,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7cd4be5bb0c74e6e8ed5d5b4968557f8 2024-11-18T20:10:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/51e32d5693234020909479353501691c is 50, key is test_row_0/B:col10/1731960653145/Put/seqid=0 2024-11-18T20:10:53,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742365_1541 (size=12001) 2024-11-18T20:10:53,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/51e32d5693234020909479353501691c 2024-11-18T20:10:53,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:53,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:53,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960713913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960713922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960713922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960713923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960713923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:53,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/1ea4b61d72f04632a8468e168a2a4a73 is 50, key is test_row_0/C:col10/1731960653145/Put/seqid=0 2024-11-18T20:10:54,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742366_1542 (size=12001) 2024-11-18T20:10:54,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/1ea4b61d72f04632a8468e168a2a4a73 2024-11-18T20:10:54,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7cd4be5bb0c74e6e8ed5d5b4968557f8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8 2024-11-18T20:10:54,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:54,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:54,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8, entries=150, sequenceid=18, filesize=30.2 K 2024-11-18T20:10:54,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/51e32d5693234020909479353501691c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/51e32d5693234020909479353501691c 2024-11-18T20:10:54,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/51e32d5693234020909479353501691c, entries=150, sequenceid=18, filesize=11.7 K 2024-11-18T20:10:54,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/1ea4b61d72f04632a8468e168a2a4a73 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/1ea4b61d72f04632a8468e168a2a4a73 2024-11-18T20:10:54,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/1ea4b61d72f04632a8468e168a2a4a73, entries=150, sequenceid=18, filesize=11.7 K 2024-11-18T20:10:54,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for fc81ed96fcb7834182497904e66e6d77 in 1011ms, sequenceid=18, compaction requested=false 2024-11-18T20:10:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:10:54,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:54,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T20:10:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:54,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:10:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:10:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d6d77590a209474580eb76d4232080ef_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960653279/Put/seqid=0 2024-11-18T20:10:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742367_1543 (size=12154) 2024-11-18T20:10:54,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:54,365 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d6d77590a209474580eb76d4232080ef_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d6d77590a209474580eb76d4232080ef_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/290c05dc7d6640f2882ad6455427cf0d, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/290c05dc7d6640f2882ad6455427cf0d is 175, key is test_row_0/A:col10/1731960653279/Put/seqid=0 2024-11-18T20:10:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742368_1544 (size=30955) 2024-11-18T20:10:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:54,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960714470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960714475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960714480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960714480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960714482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960714595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960714595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960714599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960714604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960714604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960714815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960714816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960714816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960714819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960714822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:54,832 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/290c05dc7d6640f2882ad6455427cf0d 2024-11-18T20:10:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d92055caf7f940f6a6836083bed78d6f is 50, key is test_row_0/B:col10/1731960653279/Put/seqid=0 2024-11-18T20:10:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742369_1545 (size=12001) 2024-11-18T20:10:54,894 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d92055caf7f940f6a6836083bed78d6f 2024-11-18T20:10:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/8f16feb72e124b4789acc8aa194ea303 is 50, key is test_row_0/C:col10/1731960653279/Put/seqid=0 2024-11-18T20:10:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742370_1546 (size=12001) 2024-11-18T20:10:54,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/8f16feb72e124b4789acc8aa194ea303 2024-11-18T20:10:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/290c05dc7d6640f2882ad6455427cf0d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d 2024-11-18T20:10:55,014 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-18T20:10:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d92055caf7f940f6a6836083bed78d6f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d92055caf7f940f6a6836083bed78d6f 2024-11-18T20:10:55,028 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d92055caf7f940f6a6836083bed78d6f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:10:55,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/8f16feb72e124b4789acc8aa194ea303 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/8f16feb72e124b4789acc8aa194ea303 2024-11-18T20:10:55,037 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/8f16feb72e124b4789acc8aa194ea303, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:10:55,040 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fc81ed96fcb7834182497904e66e6d77 in 794ms, sequenceid=40, compaction requested=false 2024-11-18T20:10:55,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:55,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-18T20:10:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-18T20:10:55,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-18T20:10:55,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9300 sec 2024-11-18T20:10:55,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9440 sec 2024-11-18T20:10:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:55,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:55,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:10:55,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185e22b318f2eb46b3a433f316426a7287_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:55,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742371_1547 (size=14594) 2024-11-18T20:10:55,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:55,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185e22b318f2eb46b3a433f316426a7287_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185e22b318f2eb46b3a433f316426a7287_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:55,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/6ccec64cd3ad4559826b62fde79e660a, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:55,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/6ccec64cd3ad4559826b62fde79e660a is 175, key is test_row_0/A:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960715180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960715220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960715220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960715220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960715228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742372_1548 (size=39549) 2024-11-18T20:10:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T20:10:55,270 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:55,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-18T20:10:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:55,291 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:55,299 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:55,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:55,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960715342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960715342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960715342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960715343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960715343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:55,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:55,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960715554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960715554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960715554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960715554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960715563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:55,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/6ccec64cd3ad4559826b62fde79e660a 2024-11-18T20:10:55,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/2d63cacbc55245dfb465e2dd70533ff2 is 50, key is test_row_0/B:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742373_1549 (size=12001) 2024-11-18T20:10:55,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:55,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960715863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960715863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960715863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960715863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960715873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:55,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:10:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:55,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:55,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:56,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:56,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/2d63cacbc55245dfb465e2dd70533ff2 2024-11-18T20:10:56,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/0dce9dfcd2a5492298da57440332cb8b is 50, key is test_row_0/C:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:56,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742374_1550 (size=12001) 2024-11-18T20:10:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960716372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:56,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960716372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:56,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960716372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:56,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960716382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960716382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:56,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:56,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/0dce9dfcd2a5492298da57440332cb8b 2024-11-18T20:10:56,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/6ccec64cd3ad4559826b62fde79e660a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a 2024-11-18T20:10:56,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a, entries=200, sequenceid=55, filesize=38.6 K 2024-11-18T20:10:56,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/2d63cacbc55245dfb465e2dd70533ff2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/2d63cacbc55245dfb465e2dd70533ff2 2024-11-18T20:10:56,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:56,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/2d63cacbc55245dfb465e2dd70533ff2, entries=150, sequenceid=55, filesize=11.7 K 2024-11-18T20:10:56,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:56,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/0dce9dfcd2a5492298da57440332cb8b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/0dce9dfcd2a5492298da57440332cb8b 2024-11-18T20:10:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:56,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/0dce9dfcd2a5492298da57440332cb8b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-18T20:10:56,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for fc81ed96fcb7834182497904e66e6d77 in 1658ms, sequenceid=55, compaction requested=true 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:56,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T20:10:56,785 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:56,787 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:56,792 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:56,792 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:10:56,792 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,793 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=99.1 K 2024-11-18T20:10:56,793 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,793 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a] 2024-11-18T20:10:56,800 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd4be5bb0c74e6e8ed5d5b4968557f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731960653145 2024-11-18T20:10:56,806 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 290c05dc7d6640f2882ad6455427cf0d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960653224 2024-11-18T20:10:56,807 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:56,807 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:10:56,807 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,807 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/1ea4b61d72f04632a8468e168a2a4a73, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/8f16feb72e124b4789acc8aa194ea303, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/0dce9dfcd2a5492298da57440332cb8b] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.2 K 2024-11-18T20:10:56,810 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ccec64cd3ad4559826b62fde79e660a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960654468 2024-11-18T20:10:56,811 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea4b61d72f04632a8468e168a2a4a73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731960653145 2024-11-18T20:10:56,819 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f16feb72e124b4789acc8aa194ea303, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960653224 2024-11-18T20:10:56,823 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dce9dfcd2a5492298da57440332cb8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960654468 2024-11-18T20:10:56,842 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:56,861 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:56,861 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/05265071b26e4e9b8f2de373fb1edcb3 is 50, key is test_row_0/C:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:56,863 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118da848e0880b943cdad9d50bcb864cccd_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:56,865 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118da848e0880b943cdad9d50bcb864cccd_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:56,865 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118da848e0880b943cdad9d50bcb864cccd_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:56,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-18T20:10:56,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:56,903 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:10:56,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742375_1551 (size=12104) 2024-11-18T20:10:56,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742376_1552 (size=4469) 2024-11-18T20:10:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186c457adffd0440b59f00ead62412b85f_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960655219/Put/seqid=0 2024-11-18T20:10:56,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742377_1553 (size=12154) 2024-11-18T20:10:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:57,042 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186c457adffd0440b59f00ead62412b85f_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186c457adffd0440b59f00ead62412b85f_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:57,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/4fbc1899fe1f4d56a24668ba8348d088, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:57,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/4fbc1899fe1f4d56a24668ba8348d088 is 175, key is test_row_0/A:col10/1731960655219/Put/seqid=0 2024-11-18T20:10:57,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742378_1554 (size=30955) 2024-11-18T20:10:57,104 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/4fbc1899fe1f4d56a24668ba8348d088 2024-11-18T20:10:57,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/a3176e5fe10c4bc6a7a5072a9e97cd9e is 50, key is test_row_0/B:col10/1731960655219/Put/seqid=0 2024-11-18T20:10:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742379_1555 (size=12001) 2024-11-18T20:10:57,313 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/05265071b26e4e9b8f2de373fb1edcb3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/05265071b26e4e9b8f2de373fb1edcb3 2024-11-18T20:10:57,324 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#471 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:57,324 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/24dd0284b6a643f494f2ac00506bd19c is 175, key is test_row_0/A:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:57,332 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into 05265071b26e4e9b8f2de373fb1edcb3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:57,332 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:57,332 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=13, startTime=1731960656785; duration=0sec 2024-11-18T20:10:57,332 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:57,332 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:10:57,333 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:57,336 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:57,337 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:10:57,337 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:57,337 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/51e32d5693234020909479353501691c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d92055caf7f940f6a6836083bed78d6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/2d63cacbc55245dfb465e2dd70533ff2] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.2 K 2024-11-18T20:10:57,340 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e32d5693234020909479353501691c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731960653145 2024-11-18T20:10:57,345 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d92055caf7f940f6a6836083bed78d6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960653224 2024-11-18T20:10:57,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742380_1556 (size=31058) 2024-11-18T20:10:57,351 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d63cacbc55245dfb465e2dd70533ff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960654468 2024-11-18T20:10:57,383 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:57,384 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/8e477f74ed2743f78904fc6e6bd111fc is 50, key is test_row_0/B:col10/1731960655125/Put/seqid=0 2024-11-18T20:10:57,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742381_1557 (size=12104) 2024-11-18T20:10:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:57,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960717410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960717412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960717412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960717415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960717416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:57,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960717524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960717524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960717524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960717534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960717534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,619 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/a3176e5fe10c4bc6a7a5072a9e97cd9e 2024-11-18T20:10:57,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/4275e3eb146e400b98fc3445dfa4f0b3 is 50, key is test_row_0/C:col10/1731960655219/Put/seqid=0 2024-11-18T20:10:57,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742382_1558 (size=12001) 2024-11-18T20:10:57,682 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/4275e3eb146e400b98fc3445dfa4f0b3 2024-11-18T20:10:57,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/4fbc1899fe1f4d56a24668ba8348d088 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088 2024-11-18T20:10:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960717734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960717734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960717735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960717743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960717744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:57,756 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088, entries=150, sequenceid=77, filesize=30.2 K 2024-11-18T20:10:57,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/a3176e5fe10c4bc6a7a5072a9e97cd9e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/a3176e5fe10c4bc6a7a5072a9e97cd9e 2024-11-18T20:10:57,777 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/24dd0284b6a643f494f2ac00506bd19c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/24dd0284b6a643f494f2ac00506bd19c 2024-11-18T20:10:57,793 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into 24dd0284b6a643f494f2ac00506bd19c(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-18T20:10:57,793 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:57,793 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=13, startTime=1731960656785; duration=1sec 2024-11-18T20:10:57,793 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:57,793 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:10:57,796 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/a3176e5fe10c4bc6a7a5072a9e97cd9e, entries=150, sequenceid=77, filesize=11.7 K 2024-11-18T20:10:57,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/4275e3eb146e400b98fc3445dfa4f0b3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4275e3eb146e400b98fc3445dfa4f0b3 2024-11-18T20:10:57,805 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/8e477f74ed2743f78904fc6e6bd111fc as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/8e477f74ed2743f78904fc6e6bd111fc 2024-11-18T20:10:57,811 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4275e3eb146e400b98fc3445dfa4f0b3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-18T20:10:57,812 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fc81ed96fcb7834182497904e66e6d77 in 909ms, sequenceid=77, compaction requested=false 2024-11-18T20:10:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-18T20:10:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-18T20:10:57,817 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into 8e477f74ed2743f78904fc6e6bd111fc(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:57,817 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:57,817 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=13, startTime=1731960656785; duration=0sec 2024-11-18T20:10:57,817 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:57,817 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:10:57,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-18T20:10:57,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5150 sec 2024-11-18T20:10:57,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.5470 sec 2024-11-18T20:10:58,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:58,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:10:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:58,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111833285baba3b14604a0930aa08136abee_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960718087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960718088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960718091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960718091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960718092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742383_1559 (size=17034) 2024-11-18T20:10:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960718202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960718202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960718202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960718202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960718211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960718412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960718412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960718412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960718415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960718425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:58,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111833285baba3b14604a0930aa08136abee_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111833285baba3b14604a0930aa08136abee_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:58,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a16ad30071854dc69bbd6863e29f4152, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:58,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a16ad30071854dc69bbd6863e29f4152 is 175, key is test_row_0/A:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742384_1560 (size=48139) 2024-11-18T20:10:58,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a16ad30071854dc69bbd6863e29f4152 2024-11-18T20:10:58,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/e425ac4877354635a1a5b02d43bea44f is 50, key is test_row_0/B:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742385_1561 (size=12001) 2024-11-18T20:10:58,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/e425ac4877354635a1a5b02d43bea44f 2024-11-18T20:10:58,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/901b730a779e4f818463352944738ea4 is 50, key is test_row_0/C:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742386_1562 (size=12001) 2024-11-18T20:10:58,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/901b730a779e4f818463352944738ea4 2024-11-18T20:10:58,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a16ad30071854dc69bbd6863e29f4152 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152 2024-11-18T20:10:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960718722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960718722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960718723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960718723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960718732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:58,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152, entries=250, sequenceid=96, filesize=47.0 K 2024-11-18T20:10:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/e425ac4877354635a1a5b02d43bea44f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e425ac4877354635a1a5b02d43bea44f 2024-11-18T20:10:58,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e425ac4877354635a1a5b02d43bea44f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T20:10:58,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/901b730a779e4f818463352944738ea4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/901b730a779e4f818463352944738ea4 2024-11-18T20:10:58,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/901b730a779e4f818463352944738ea4, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T20:10:58,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for fc81ed96fcb7834182497904e66e6d77 in 709ms, sequenceid=96, compaction requested=true 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:10:58,755 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:58,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:10:58,756 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:58,756 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:58,756 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:10:58,757 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:58,757 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/8e477f74ed2743f78904fc6e6bd111fc, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/a3176e5fe10c4bc6a7a5072a9e97cd9e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e425ac4877354635a1a5b02d43bea44f] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.3 K 2024-11-18T20:10:58,757 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:58,757 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:10:58,757 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:58,757 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/24dd0284b6a643f494f2ac00506bd19c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=107.6 K 2024-11-18T20:10:58,757 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:58,757 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/24dd0284b6a643f494f2ac00506bd19c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152] 2024-11-18T20:10:58,757 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e477f74ed2743f78904fc6e6bd111fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960654468 2024-11-18T20:10:58,757 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24dd0284b6a643f494f2ac00506bd19c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960654468 2024-11-18T20:10:58,758 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a3176e5fe10c4bc6a7a5072a9e97cd9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731960655152 2024-11-18T20:10:58,758 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbc1899fe1f4d56a24668ba8348d088, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731960655152 2024-11-18T20:10:58,758 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e425ac4877354635a1a5b02d43bea44f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731960657408 2024-11-18T20:10:58,758 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16ad30071854dc69bbd6863e29f4152, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731960657408 2024-11-18T20:10:58,768 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:58,769 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d0cd8f5fbc5d4bedbaf393ff20530f9e is 50, key is test_row_0/B:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,772 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:58,774 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411184547798acfee4dc8a2ff5e98085c8557_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:58,776 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411184547798acfee4dc8a2ff5e98085c8557_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:58,776 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184547798acfee4dc8a2ff5e98085c8557_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:58,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742387_1563 (size=12207) 2024-11-18T20:10:58,819 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d0cd8f5fbc5d4bedbaf393ff20530f9e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d0cd8f5fbc5d4bedbaf393ff20530f9e 2024-11-18T20:10:58,825 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into d0cd8f5fbc5d4bedbaf393ff20530f9e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:58,825 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:58,825 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=13, startTime=1731960658755; duration=0sec 2024-11-18T20:10:58,826 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:10:58,826 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:10:58,826 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:10:58,826 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:10:58,828 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:10:58,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742388_1564 (size=4469) 2024-11-18T20:10:58,830 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:58,830 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/05265071b26e4e9b8f2de373fb1edcb3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4275e3eb146e400b98fc3445dfa4f0b3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/901b730a779e4f818463352944738ea4] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.3 K 2024-11-18T20:10:58,831 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 05265071b26e4e9b8f2de373fb1edcb3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960654468 2024-11-18T20:10:58,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4275e3eb146e400b98fc3445dfa4f0b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731960655152 2024-11-18T20:10:58,832 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 901b730a779e4f818463352944738ea4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731960657408 2024-11-18T20:10:58,837 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#481 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:58,837 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/0302c19a2aae48aabb02eb4b6ce253e6 is 175, key is test_row_0/A:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,845 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:10:58,846 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/6afac72c98a94f328fb108adacf72e21 is 50, key is test_row_0/C:col10/1731960658044/Put/seqid=0 2024-11-18T20:10:58,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742389_1565 (size=31161) 2024-11-18T20:10:58,904 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/0302c19a2aae48aabb02eb4b6ce253e6 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/0302c19a2aae48aabb02eb4b6ce253e6 2024-11-18T20:10:58,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742390_1566 (size=12207) 2024-11-18T20:10:58,918 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into 0302c19a2aae48aabb02eb4b6ce253e6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:58,918 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:58,918 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=13, startTime=1731960658755; duration=0sec 2024-11-18T20:10:58,918 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:58,918 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:10:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:59,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:59,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:10:59,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f246c13c74aa44cbb84de61b90d318c2_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960658086/Put/seqid=0 2024-11-18T20:10:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960719256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960719258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960719258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960719258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960719258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742391_1567 (size=12154) 2024-11-18T20:10:59,295 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:59,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f246c13c74aa44cbb84de61b90d318c2_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f246c13c74aa44cbb84de61b90d318c2_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:59,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/25746ed8a8d74639a2c033e8623b2beb, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:59,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/25746ed8a8d74639a2c033e8623b2beb is 175, key is test_row_0/A:col10/1731960658086/Put/seqid=0 2024-11-18T20:10:59,327 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/6afac72c98a94f328fb108adacf72e21 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6afac72c98a94f328fb108adacf72e21 2024-11-18T20:10:59,333 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into 6afac72c98a94f328fb108adacf72e21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:10:59,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:59,333 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=13, startTime=1731960658755; duration=0sec 2024-11-18T20:10:59,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:10:59,333 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:10:59,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742392_1568 (size=30955) 2024-11-18T20:10:59,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/25746ed8a8d74639a2c033e8623b2beb 2024-11-18T20:10:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960719372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960719372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960719375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960719375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/642b0675cd2d440580dcb02d8dcf70e3 is 50, key is test_row_0/B:col10/1731960658086/Put/seqid=0 2024-11-18T20:10:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960719372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742393_1569 (size=12001) 2024-11-18T20:10:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/642b0675cd2d440580dcb02d8dcf70e3 2024-11-18T20:10:59,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/4e93b7a8a5114e9fbfa6cfc18603e1cd is 50, key is test_row_0/C:col10/1731960658086/Put/seqid=0 2024-11-18T20:10:59,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742394_1570 (size=12001) 2024-11-18T20:10:59,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/4e93b7a8a5114e9fbfa6cfc18603e1cd 2024-11-18T20:10:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T20:10:59,467 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:10:59,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:10:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/25746ed8a8d74639a2c033e8623b2beb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb 2024-11-18T20:10:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-18T20:10:59,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:10:59,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:10:59,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:10:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:10:59,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb, entries=150, sequenceid=121, filesize=30.2 K 2024-11-18T20:10:59,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/642b0675cd2d440580dcb02d8dcf70e3 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/642b0675cd2d440580dcb02d8dcf70e3 2024-11-18T20:10:59,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/642b0675cd2d440580dcb02d8dcf70e3, entries=150, sequenceid=121, filesize=11.7 K 2024-11-18T20:10:59,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/4e93b7a8a5114e9fbfa6cfc18603e1cd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4e93b7a8a5114e9fbfa6cfc18603e1cd 2024-11-18T20:10:59,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4e93b7a8a5114e9fbfa6cfc18603e1cd, entries=150, sequenceid=121, filesize=11.7 K 2024-11-18T20:10:59,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for fc81ed96fcb7834182497904e66e6d77 in 299ms, sequenceid=121, compaction requested=false 2024-11-18T20:10:59,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:10:59,591 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:10:59,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:10:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:10:59,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188aa4adcc41cb4b79bf7a7a17168e4639_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960659587/Put/seqid=0 2024-11-18T20:10:59,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:10:59,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:59,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:59,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:59,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960719649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742395_1571 (size=12304) 2024-11-18T20:10:59,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960719650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960719653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960719654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960719656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,664 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:10:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188aa4adcc41cb4b79bf7a7a17168e4639_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188aa4adcc41cb4b79bf7a7a17168e4639_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:10:59,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/03be40cdaf31466ba0657cc7d46eabd4, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:10:59,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/03be40cdaf31466ba0657cc7d46eabd4 is 175, key is test_row_0/A:col10/1731960659587/Put/seqid=0 2024-11-18T20:10:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742396_1572 (size=31105) 2024-11-18T20:10:59,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960719767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960719772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960719772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960719772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960719773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:10:59,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:59,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:59,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:59,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:10:59,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:10:59,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:59,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:10:59,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:10:59,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:10:59,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960719971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960719982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960719983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960719983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:10:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960719984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:10:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:10:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:10:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,095 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/03be40cdaf31466ba0657cc7d46eabd4 2024-11-18T20:11:00,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/321b04be93df4a038b7d0a484c90f223 is 50, key is test_row_0/B:col10/1731960659587/Put/seqid=0 2024-11-18T20:11:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:11:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742397_1573 (size=12151) 2024-11-18T20:11:00,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960720283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960720292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960720293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960720293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960720293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/321b04be93df4a038b7d0a484c90f223 2024-11-18T20:11:00,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/30e1304212404c6cbda5c928490dddbd is 50, key is test_row_0/C:col10/1731960659587/Put/seqid=0 2024-11-18T20:11:00,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742398_1574 (size=12151) 2024-11-18T20:11:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:11:00,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960720795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960720802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960720803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960720804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960720811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:00,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:00,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:01,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/30e1304212404c6cbda5c928490dddbd 2024-11-18T20:11:01,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/03be40cdaf31466ba0657cc7d46eabd4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4 2024-11-18T20:11:01,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4, entries=150, sequenceid=137, filesize=30.4 K 2024-11-18T20:11:01,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/321b04be93df4a038b7d0a484c90f223 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/321b04be93df4a038b7d0a484c90f223 2024-11-18T20:11:01,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/321b04be93df4a038b7d0a484c90f223, entries=150, sequenceid=137, filesize=11.9 K 2024-11-18T20:11:01,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/30e1304212404c6cbda5c928490dddbd as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/30e1304212404c6cbda5c928490dddbd 2024-11-18T20:11:01,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/30e1304212404c6cbda5c928490dddbd, entries=150, sequenceid=137, filesize=11.9 K 2024-11-18T20:11:01,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fc81ed96fcb7834182497904e66e6d77 in 1448ms, sequenceid=137, compaction requested=true 2024-11-18T20:11:01,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:01,040 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:01,041 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:01,041 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:11:01,041 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:01,041 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/0302c19a2aae48aabb02eb4b6ce253e6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=91.0 K 2024-11-18T20:11:01,041 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:01,042 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/0302c19a2aae48aabb02eb4b6ce253e6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4] 2024-11-18T20:11:01,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:11:01,042 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0302c19a2aae48aabb02eb4b6ce253e6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731960657408 2024-11-18T20:11:01,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:01,042 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25746ed8a8d74639a2c033e8623b2beb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731960658086 2024-11-18T20:11:01,042 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:01,043 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03be40cdaf31466ba0657cc7d46eabd4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960659256 2024-11-18T20:11:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:11:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:01,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:11:01,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:01,046 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:01,046 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:11:01,046 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:01,046 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d0cd8f5fbc5d4bedbaf393ff20530f9e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/642b0675cd2d440580dcb02d8dcf70e3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/321b04be93df4a038b7d0a484c90f223] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.5 K 2024-11-18T20:11:01,047 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cd8f5fbc5d4bedbaf393ff20530f9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731960657408 2024-11-18T20:11:01,047 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 642b0675cd2d440580dcb02d8dcf70e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731960658086 2024-11-18T20:11:01,048 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 321b04be93df4a038b7d0a484c90f223, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960659256 2024-11-18T20:11:01,056 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:01,061 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:01,062 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/69c6371a826744779db02b71d637ff1d is 50, key is test_row_0/B:col10/1731960659587/Put/seqid=0 2024-11-18T20:11:01,073 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118adf0a293a67f488593680f9f9861fcf7_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:01,075 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118adf0a293a67f488593680f9f9861fcf7_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:01,075 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118adf0a293a67f488593680f9f9861fcf7_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:01,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:11:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:01,081 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:11:01,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742399_1575 (size=12459) 2024-11-18T20:11:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183b85214f123243fab7e17f6815d98e9d_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960659638/Put/seqid=0 2024-11-18T20:11:01,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742400_1576 (size=4469) 2024-11-18T20:11:01,152 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#489 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:01,152 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/de33e6b031a44668b1b8a6a47395769e is 175, key is test_row_0/A:col10/1731960659587/Put/seqid=0 2024-11-18T20:11:01,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742401_1577 (size=12304) 2024-11-18T20:11:01,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:01,195 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183b85214f123243fab7e17f6815d98e9d_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183b85214f123243fab7e17f6815d98e9d_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:01,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742402_1578 (size=31413) 2024-11-18T20:11:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/ae08defe3baa41efb9b41fecbfbafe70, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:01,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/ae08defe3baa41efb9b41fecbfbafe70 is 175, key is test_row_0/A:col10/1731960659638/Put/seqid=0 2024-11-18T20:11:01,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742403_1579 (size=31105) 2024-11-18T20:11:01,242 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/ae08defe3baa41efb9b41fecbfbafe70 2024-11-18T20:11:01,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/52fee884b8c346589c424245718be3cb is 50, key is test_row_0/B:col10/1731960659638/Put/seqid=0 2024-11-18T20:11:01,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742404_1580 (size=12151) 2024-11-18T20:11:01,550 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/69c6371a826744779db02b71d637ff1d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/69c6371a826744779db02b71d637ff1d 2024-11-18T20:11:01,568 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into 69c6371a826744779db02b71d637ff1d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:01,568 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:01,568 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=13, startTime=1731960661042; duration=0sec 2024-11-18T20:11:01,569 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:01,569 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:11:01,569 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:01,572 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:01,572 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:11:01,572 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:01,572 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6afac72c98a94f328fb108adacf72e21, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4e93b7a8a5114e9fbfa6cfc18603e1cd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/30e1304212404c6cbda5c928490dddbd] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.5 K 2024-11-18T20:11:01,573 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6afac72c98a94f328fb108adacf72e21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731960657408 2024-11-18T20:11:01,582 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e93b7a8a5114e9fbfa6cfc18603e1cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731960658086 2024-11-18T20:11:01,585 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e1304212404c6cbda5c928490dddbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960659256 2024-11-18T20:11:01,610 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:01,610 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/79381ecf91834f13b4e353f73820ff85 is 50, key is test_row_0/C:col10/1731960659587/Put/seqid=0 2024-11-18T20:11:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:11:01,647 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/de33e6b031a44668b1b8a6a47395769e as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/de33e6b031a44668b1b8a6a47395769e 2024-11-18T20:11:01,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742405_1581 (size=12459) 2024-11-18T20:11:01,687 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into de33e6b031a44668b1b8a6a47395769e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:01,687 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:01,687 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=13, startTime=1731960661040; duration=0sec 2024-11-18T20:11:01,688 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:01,688 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:11:01,731 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/52fee884b8c346589c424245718be3cb 2024-11-18T20:11:01,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/ecc8a05265164487be7da9c2aab79c49 is 50, key is test_row_0/C:col10/1731960659638/Put/seqid=0 2024-11-18T20:11:01,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742406_1582 (size=12151) 2024-11-18T20:11:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:01,831 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/ecc8a05265164487be7da9c2aab79c49 2024-11-18T20:11:01,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960721863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960721865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960721865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960721866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/ae08defe3baa41efb9b41fecbfbafe70 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70 2024-11-18T20:11:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960721875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70, entries=150, sequenceid=158, filesize=30.4 K 2024-11-18T20:11:01,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/52fee884b8c346589c424245718be3cb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/52fee884b8c346589c424245718be3cb 2024-11-18T20:11:01,932 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/52fee884b8c346589c424245718be3cb, entries=150, sequenceid=158, filesize=11.9 K 2024-11-18T20:11:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/ecc8a05265164487be7da9c2aab79c49 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ecc8a05265164487be7da9c2aab79c49 2024-11-18T20:11:01,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960721982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960721983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960721983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960721983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960721995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,004 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ecc8a05265164487be7da9c2aab79c49, entries=150, sequenceid=158, filesize=11.9 K 2024-11-18T20:11:02,011 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for fc81ed96fcb7834182497904e66e6d77 in 930ms, sequenceid=158, compaction requested=false 2024-11-18T20:11:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-18T20:11:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-18T20:11:02,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-18T20:11:02,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5410 sec 2024-11-18T20:11:02,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.5670 sec 2024-11-18T20:11:02,147 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/79381ecf91834f13b4e353f73820ff85 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/79381ecf91834f13b4e353f73820ff85 2024-11-18T20:11:02,197 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into 79381ecf91834f13b4e353f73820ff85(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:02,197 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:02,197 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=13, startTime=1731960661044; duration=0sec 2024-11-18T20:11:02,198 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:02,198 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:11:02,198 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:11:02,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:11:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960722223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960722224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960722227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960722228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960722230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111804db54f8b0fa4bfd9b2ea673714974ca_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:02,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742407_1583 (size=12304) 2024-11-18T20:11:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960722342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960722342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960722343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960722343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960722343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960722552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960722554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960722555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960722563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960722563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:02,720 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111804db54f8b0fa4bfd9b2ea673714974ca_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111804db54f8b0fa4bfd9b2ea673714974ca_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:02,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e91dab29a7c34b7b850b50eabb675a68, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:02,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e91dab29a7c34b7b850b50eabb675a68 is 175, key is test_row_0/A:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:02,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742408_1584 (size=31105) 2024-11-18T20:11:02,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960722863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960722864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960722864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960722872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960722872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e91dab29a7c34b7b850b50eabb675a68 2024-11-18T20:11:03,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d123b579bd3c416f8a9a5c00c50a76f1 is 50, key is test_row_0/B:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:03,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742409_1585 (size=12151) 2024-11-18T20:11:03,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d123b579bd3c416f8a9a5c00c50a76f1 2024-11-18T20:11:03,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/af21f837501548efb36ee3351ad8a281 is 50, key is test_row_0/C:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:03,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742410_1586 (size=12151) 2024-11-18T20:11:03,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/af21f837501548efb36ee3351ad8a281 2024-11-18T20:11:03,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e91dab29a7c34b7b850b50eabb675a68 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68 2024-11-18T20:11:03,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68, entries=150, sequenceid=179, filesize=30.4 K 2024-11-18T20:11:03,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d123b579bd3c416f8a9a5c00c50a76f1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d123b579bd3c416f8a9a5c00c50a76f1 2024-11-18T20:11:03,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d123b579bd3c416f8a9a5c00c50a76f1, entries=150, sequenceid=179, filesize=11.9 K 2024-11-18T20:11:03,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/af21f837501548efb36ee3351ad8a281 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/af21f837501548efb36ee3351ad8a281 2024-11-18T20:11:03,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/af21f837501548efb36ee3351ad8a281, entries=150, sequenceid=179, filesize=11.9 K 2024-11-18T20:11:03,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for fc81ed96fcb7834182497904e66e6d77 in 1178ms, sequenceid=179, compaction requested=true 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:03,377 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:03,377 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:11:03,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:11:03,389 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,389 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,389 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/69c6371a826744779db02b71d637ff1d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/52fee884b8c346589c424245718be3cb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d123b579bd3c416f8a9a5c00c50a76f1] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.9 K 2024-11-18T20:11:03,389 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/de33e6b031a44668b1b8a6a47395769e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=91.4 K 2024-11-18T20:11:03,389 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/de33e6b031a44668b1b8a6a47395769e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68] 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c6371a826744779db02b71d637ff1d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960659256 2024-11-18T20:11:03,389 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting de33e6b031a44668b1b8a6a47395769e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960659256 2024-11-18T20:11:03,390 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae08defe3baa41efb9b41fecbfbafe70, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731960659638 2024-11-18T20:11:03,390 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 52fee884b8c346589c424245718be3cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731960659638 2024-11-18T20:11:03,390 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91dab29a7c34b7b850b50eabb675a68, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731960661862 2024-11-18T20:11:03,391 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d123b579bd3c416f8a9a5c00c50a76f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731960661862 2024-11-18T20:11:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:03,396 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:11:03,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:11:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960723420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960723421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960723424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960723429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960723430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,443 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:03,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111847df8a220e104b6d8fc00c9cdd0bdeaf_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960662221/Put/seqid=0 2024-11-18T20:11:03,465 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:03,466 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/9a24b680c01f4a77bc9d1040cee7744a is 50, key is test_row_0/B:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:03,468 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411183873b65568764f9599b20c9d26e0eea8_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:03,470 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411183873b65568764f9599b20c9d26e0eea8_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:03,470 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183873b65568764f9599b20c9d26e0eea8_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:03,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742412_1588 (size=12561) 2024-11-18T20:11:03,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742413_1589 (size=4469) 2024-11-18T20:11:03,524 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#498 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:03,524 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/ef28a5dc5ab7492ba5eda390be530362 is 175, key is test_row_0/A:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742411_1587 (size=12304) 2024-11-18T20:11:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960723533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960723533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,543 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/9a24b680c01f4a77bc9d1040cee7744a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/9a24b680c01f4a77bc9d1040cee7744a 2024-11-18T20:11:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960723542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960723542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960723543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,549 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into 9a24b680c01f4a77bc9d1040cee7744a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:03,549 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:03,549 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=13, startTime=1731960663377; duration=0sec 2024-11-18T20:11:03,549 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:03,549 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:11:03,549 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:03,550 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:03,550 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:11:03,550 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,550 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/79381ecf91834f13b4e353f73820ff85, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ecc8a05265164487be7da9c2aab79c49, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/af21f837501548efb36ee3351ad8a281] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=35.9 K 2024-11-18T20:11:03,552 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 79381ecf91834f13b4e353f73820ff85, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960659256 2024-11-18T20:11:03,552 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc8a05265164487be7da9c2aab79c49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731960659638 2024-11-18T20:11:03,552 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting af21f837501548efb36ee3351ad8a281, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731960661862 2024-11-18T20:11:03,569 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:03,570 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/b558fe2fbf674445bd234343855db8f2 is 50, key is test_row_0/C:col10/1731960662198/Put/seqid=0 2024-11-18T20:11:03,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742414_1590 (size=31515) 2024-11-18T20:11:03,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742415_1591 (size=12561) 2024-11-18T20:11:03,621 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/b558fe2fbf674445bd234343855db8f2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/b558fe2fbf674445bd234343855db8f2 2024-11-18T20:11:03,641 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into b558fe2fbf674445bd234343855db8f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:03,641 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:03,641 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=13, startTime=1731960663377; duration=0sec 2024-11-18T20:11:03,641 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:03,641 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:11:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:11:03,655 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:11:03,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:11:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-18T20:11:03,657 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:11:03,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:11:03,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:11:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:11:03,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960723742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960723742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960723755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960723755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960723757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:11:03,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:11:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:03,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:03,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:03,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:11:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:11:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:03,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:03,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111847df8a220e104b6d8fc00c9cdd0bdeaf_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111847df8a220e104b6d8fc00c9cdd0bdeaf_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:03,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/c953dc9575514e68864877b332dc4d63, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:03,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/c953dc9575514e68864877b332dc4d63 is 175, key is test_row_0/A:col10/1731960662221/Put/seqid=0 2024-11-18T20:11:04,005 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/ef28a5dc5ab7492ba5eda390be530362 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ef28a5dc5ab7492ba5eda390be530362 2024-11-18T20:11:04,010 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into ef28a5dc5ab7492ba5eda390be530362(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:04,010 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:04,010 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=13, startTime=1731960663377; duration=0sec 2024-11-18T20:11:04,010 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:04,010 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:11:04,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742416_1592 (size=31105) 2024-11-18T20:11:04,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/c953dc9575514e68864877b332dc4d63 2024-11-18T20:11:04,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d3d947e225f84867bf2bb1c3f9cf824b is 50, key is test_row_0/B:col10/1731960662221/Put/seqid=0 2024-11-18T20:11:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960724052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960724052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960724062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960724062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960724062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742417_1593 (size=12151) 2024-11-18T20:11:04,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d3d947e225f84867bf2bb1c3f9cf824b 2024-11-18T20:11:04,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/9fdd8afb6200490e90a42ecc65a5986f is 50, key is test_row_0/C:col10/1731960662221/Put/seqid=0 2024-11-18T20:11:04,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:11:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:04,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:04,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:04,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742418_1594 (size=12151) 2024-11-18T20:11:04,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/9fdd8afb6200490e90a42ecc65a5986f 2024-11-18T20:11:04,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/c953dc9575514e68864877b332dc4d63 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63 2024-11-18T20:11:04,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63, entries=150, sequenceid=200, filesize=30.4 K 2024-11-18T20:11:04,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/d3d947e225f84867bf2bb1c3f9cf824b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d3d947e225f84867bf2bb1c3f9cf824b 2024-11-18T20:11:04,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d3d947e225f84867bf2bb1c3f9cf824b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-18T20:11:04,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/9fdd8afb6200490e90a42ecc65a5986f as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/9fdd8afb6200490e90a42ecc65a5986f 2024-11-18T20:11:04,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/9fdd8afb6200490e90a42ecc65a5986f, entries=150, sequenceid=200, filesize=11.9 K 2024-11-18T20:11:04,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for fc81ed96fcb7834182497904e66e6d77 in 789ms, sequenceid=200, compaction requested=false 2024-11-18T20:11:04,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:04,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:11:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:11:04,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:04,283 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:11:04,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843f8fcc02cd449c198575bb4288d4410_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960663427/Put/seqid=0 2024-11-18T20:11:04,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742419_1595 (size=12304) 2024-11-18T20:11:04,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960724597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960724598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960724604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960724605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960724607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960724714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960724714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960724724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960724724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960724724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:04,768 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843f8fcc02cd449c198575bb4288d4410_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843f8fcc02cd449c198575bb4288d4410_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:04,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e0e0bdd48dad4f519ec2d80161cd275b, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:04,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e0e0bdd48dad4f519ec2d80161cd275b is 175, key is test_row_0/A:col10/1731960663427/Put/seqid=0 2024-11-18T20:11:04,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742420_1596 (size=31105) 2024-11-18T20:11:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:11:04,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960724927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960724932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960724934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960724942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960724943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,192 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e0e0bdd48dad4f519ec2d80161cd275b 2024-11-18T20:11:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/3c46ff30bdd547eda4f4c3b4ccf55963 is 50, key is test_row_0/B:col10/1731960663427/Put/seqid=0 2024-11-18T20:11:05,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742421_1597 (size=12151) 2024-11-18T20:11:05,238 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/3c46ff30bdd547eda4f4c3b4ccf55963 2024-11-18T20:11:05,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960725242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960725243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/ad177fb6e5164c94bab0461488c5a50d is 50, key is test_row_0/C:col10/1731960663427/Put/seqid=0 2024-11-18T20:11:05,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960725252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960725252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960725262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742422_1598 (size=12151) 2024-11-18T20:11:05,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/ad177fb6e5164c94bab0461488c5a50d 2024-11-18T20:11:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/e0e0bdd48dad4f519ec2d80161cd275b as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b 2024-11-18T20:11:05,295 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b, entries=150, sequenceid=219, filesize=30.4 K 2024-11-18T20:11:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/3c46ff30bdd547eda4f4c3b4ccf55963 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3c46ff30bdd547eda4f4c3b4ccf55963 2024-11-18T20:11:05,303 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3c46ff30bdd547eda4f4c3b4ccf55963, entries=150, sequenceid=219, filesize=11.9 K 2024-11-18T20:11:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/ad177fb6e5164c94bab0461488c5a50d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ad177fb6e5164c94bab0461488c5a50d 2024-11-18T20:11:05,315 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ad177fb6e5164c94bab0461488c5a50d, entries=150, sequenceid=219, filesize=11.9 K 2024-11-18T20:11:05,330 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for fc81ed96fcb7834182497904e66e6d77 in 1047ms, sequenceid=219, compaction requested=true 2024-11-18T20:11:05,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:05,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:05,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-18T20:11:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-18T20:11:05,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-18T20:11:05,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6810 sec 2024-11-18T20:11:05,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.6860 sec 2024-11-18T20:11:05,758 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:11:05,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:11:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:05,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181be119c152f84704835019555c270233_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:05,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742423_1599 (size=14794) 2024-11-18T20:11:05,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960725782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960725787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960725787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960725788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960725790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:11:05,806 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:11:05,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:11:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-18T20:11:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:05,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:11:05,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:11:05,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:11:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960725902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960725903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960725903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960725903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960725915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:05,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:05,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960726112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960726112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960726113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960726113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960726124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:06,216 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181be119c152f84704835019555c270233_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181be119c152f84704835019555c270233_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:06,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/50b34be46a9842b285232fb548468ca1, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:06,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/50b34be46a9842b285232fb548468ca1 is 175, key is test_row_0/A:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:06,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742424_1600 (size=39749) 2024-11-18T20:11:06,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960726421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960726421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960726422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960726422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:06,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960726442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/50b34be46a9842b285232fb548468ca1 2024-11-18T20:11:06,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/29c79633f4c34f9e8aca2d51a714f553 is 50, key is test_row_0/B:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:06,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742425_1601 (size=12151) 2024-11-18T20:11:06,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:06,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960726932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960726932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960726932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960726932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:06,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960726951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:07,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/29c79633f4c34f9e8aca2d51a714f553 2024-11-18T20:11:07,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/5a719077f8454b058e3d9c4fc35c3ef0 is 50, key is test_row_0/C:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:07,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742426_1602 (size=12151) 2024-11-18T20:11:07,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:07,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:07,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:07,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:07,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:07,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/5a719077f8454b058e3d9c4fc35c3ef0 2024-11-18T20:11:07,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/50b34be46a9842b285232fb548468ca1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1 2024-11-18T20:11:07,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1, entries=200, sequenceid=240, filesize=38.8 K 2024-11-18T20:11:07,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/29c79633f4c34f9e8aca2d51a714f553 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/29c79633f4c34f9e8aca2d51a714f553 2024-11-18T20:11:07,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/29c79633f4c34f9e8aca2d51a714f553, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T20:11:07,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/5a719077f8454b058e3d9c4fc35c3ef0 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/5a719077f8454b058e3d9c4fc35c3ef0 2024-11-18T20:11:07,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/5a719077f8454b058e3d9c4fc35c3ef0, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T20:11:07,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fc81ed96fcb7834182497904e66e6d77 in 1724ms, sequenceid=240, compaction requested=true 2024-11-18T20:11:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:11:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:07,482 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:11:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:11:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:07,483 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:11:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:11:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:07,483 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:11:07,483 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:11:07,483 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:11:07,483 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:11:07,483 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,483 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,484 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/9a24b680c01f4a77bc9d1040cee7744a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d3d947e225f84867bf2bb1c3f9cf824b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3c46ff30bdd547eda4f4c3b4ccf55963, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/29c79633f4c34f9e8aca2d51a714f553] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=47.9 K 2024-11-18T20:11:07,484 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ef28a5dc5ab7492ba5eda390be530362, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=130.3 K 2024-11-18T20:11:07,484 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,484 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ef28a5dc5ab7492ba5eda390be530362, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1] 2024-11-18T20:11:07,484 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a24b680c01f4a77bc9d1040cee7744a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731960661862 2024-11-18T20:11:07,484 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef28a5dc5ab7492ba5eda390be530362, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731960661862 2024-11-18T20:11:07,484 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d947e225f84867bf2bb1c3f9cf824b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731960662221 2024-11-18T20:11:07,484 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c953dc9575514e68864877b332dc4d63, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731960662221 2024-11-18T20:11:07,485 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e0bdd48dad4f519ec2d80161cd275b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731960663419 2024-11-18T20:11:07,485 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c46ff30bdd547eda4f4c3b4ccf55963, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731960663419 2024-11-18T20:11:07,485 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b34be46a9842b285232fb548468ca1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960664597 2024-11-18T20:11:07,485 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c79633f4c34f9e8aca2d51a714f553, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960664597 2024-11-18T20:11:07,490 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:07,492 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:07,492 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180a7dd76e89fa4d69b1f5b4a8c4b377a6_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:07,492 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/dcb536f5cd7b4a349488b401eef856a4 is 50, key is test_row_0/B:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:07,494 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180a7dd76e89fa4d69b1f5b4a8c4b377a6_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:07,495 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180a7dd76e89fa4d69b1f5b4a8c4b377a6_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:07,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:11:07,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,496 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:11:07,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742427_1603 (size=12697) 2024-11-18T20:11:07,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742428_1604 (size=4469) 2024-11-18T20:11:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181a01e0d16d4c40d29c2a79571a13c5a1_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960665787/Put/seqid=0 2024-11-18T20:11:07,506 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/dcb536f5cd7b4a349488b401eef856a4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/dcb536f5cd7b4a349488b401eef856a4 2024-11-18T20:11:07,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742429_1605 (size=12304) 2024-11-18T20:11:07,510 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into dcb536f5cd7b4a349488b401eef856a4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:07,510 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:07,510 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=12, startTime=1731960667482; duration=0sec 2024-11-18T20:11:07,510 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:07,510 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:11:07,510 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:11:07,511 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:11:07,511 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:11:07,511 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:07,511 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/b558fe2fbf674445bd234343855db8f2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/9fdd8afb6200490e90a42ecc65a5986f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ad177fb6e5164c94bab0461488c5a50d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/5a719077f8454b058e3d9c4fc35c3ef0] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=47.9 K 2024-11-18T20:11:07,511 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting b558fe2fbf674445bd234343855db8f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731960661862 2024-11-18T20:11:07,511 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fdd8afb6200490e90a42ecc65a5986f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731960662221 2024-11-18T20:11:07,512 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting ad177fb6e5164c94bab0461488c5a50d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731960663419 2024-11-18T20:11:07,512 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a719077f8454b058e3d9c4fc35c3ef0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960664597 2024-11-18T20:11:07,520 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:07,520 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/a62770871eb542899fa4e24bcbac0376 is 50, key is test_row_0/C:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742430_1606 (size=12697) 2024-11-18T20:11:07,527 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/a62770871eb542899fa4e24bcbac0376 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/a62770871eb542899fa4e24bcbac0376 2024-11-18T20:11:07,533 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into a62770871eb542899fa4e24bcbac0376(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:07,533 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:07,533 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=12, startTime=1731960667482; duration=0sec 2024-11-18T20:11:07,533 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:07,533 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:11:07,903 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#511 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:07,904 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/c3a43027969f433ea47634a4cfa6d306 is 175, key is test_row_0/A:col10/1731960665753/Put/seqid=0 2024-11-18T20:11:07,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742431_1607 (size=31651) 2024-11-18T20:11:07,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:07,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181a01e0d16d4c40d29c2a79571a13c5a1_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a01e0d16d4c40d29c2a79571a13c5a1_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:07,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/f3e1e633eddc479ca994261d4d23517c, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:07,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/f3e1e633eddc479ca994261d4d23517c is 175, key is test_row_0/A:col10/1731960665787/Put/seqid=0 2024-11-18T20:11:07,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742432_1608 (size=31105) 2024-11-18T20:11:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960727960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960727960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960727962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:07,965 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in fc81ed96fcb7834182497904e66e6d77 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:11:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:07,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960727964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960727964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960728072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960728072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960728072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960728072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960728282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960728282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960728282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960728282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,311 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/c3a43027969f433ea47634a4cfa6d306 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c3a43027969f433ea47634a4cfa6d306 2024-11-18T20:11:08,314 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into c3a43027969f433ea47634a4cfa6d306(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:08,314 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:08,314 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=12, startTime=1731960667482; duration=0sec 2024-11-18T20:11:08,314 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:08,314 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:11:08,316 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/f3e1e633eddc479ca994261d4d23517c 2024-11-18T20:11:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/cafe774bbf9549fa95e5e674b1286aec is 50, key is test_row_0/B:col10/1731960665787/Put/seqid=0 2024-11-18T20:11:08,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742433_1609 (size=12151) 2024-11-18T20:11:08,324 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/cafe774bbf9549fa95e5e674b1286aec 2024-11-18T20:11:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/38bc89d1988c4dca9892c6d4ff85c4b2 is 50, key is test_row_0/C:col10/1731960665787/Put/seqid=0 2024-11-18T20:11:08,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742434_1610 (size=12151) 2024-11-18T20:11:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960728593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960728593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960728594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960728594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:08,733 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/38bc89d1988c4dca9892c6d4ff85c4b2 2024-11-18T20:11:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/f3e1e633eddc479ca994261d4d23517c as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c 2024-11-18T20:11:08,742 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c, entries=150, sequenceid=255, filesize=30.4 K 2024-11-18T20:11:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/cafe774bbf9549fa95e5e674b1286aec as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/cafe774bbf9549fa95e5e674b1286aec 2024-11-18T20:11:08,749 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/cafe774bbf9549fa95e5e674b1286aec, entries=150, sequenceid=255, filesize=11.9 K 2024-11-18T20:11:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/38bc89d1988c4dca9892c6d4ff85c4b2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/38bc89d1988c4dca9892c6d4ff85c4b2 2024-11-18T20:11:08,753 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/38bc89d1988c4dca9892c6d4ff85c4b2, entries=150, sequenceid=255, filesize=11.9 K 2024-11-18T20:11:08,753 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for fc81ed96fcb7834182497904e66e6d77 in 1257ms, sequenceid=255, compaction requested=false 2024-11-18T20:11:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-18T20:11:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-18T20:11:08,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-18T20:11:08,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9450 sec 2024-11-18T20:11:08,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.9510 sec 2024-11-18T20:11:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:09,104 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:11:09,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:11:09,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118829dba8b58694d86b45e9714b31cb380_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:09,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742435_1611 (size=12454) 2024-11-18T20:11:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960729149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960729151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960729151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960729152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960729262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960729262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960729262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960729262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960729472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960729472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960729472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960729472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:09,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118829dba8b58694d86b45e9714b31cb380_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118829dba8b58694d86b45e9714b31cb380_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:09,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/99813ef05a7448e9aaf4f59b7d4d77b1, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:09,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/99813ef05a7448e9aaf4f59b7d4d77b1 is 175, key is test_row_0/A:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:09,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742436_1612 (size=31255) 2024-11-18T20:11:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960729781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960729781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960729782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960729782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:09,929 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/99813ef05a7448e9aaf4f59b7d4d77b1 2024-11-18T20:11:09,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/e3d5e17f9ec84c6d96debed5c35ef9a5 is 50, key is test_row_0/B:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:09,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742437_1613 (size=12301) 2024-11-18T20:11:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:11:09,962 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:11:09,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:11:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-18T20:11:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:09,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:11:09,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:11:09,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:11:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54858 deadline: 1731960729981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:09,985 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d8f95bb18b58,42447,1731960497538 for 'test_row_0' in fc81ed96fcb7834182497904e66e6d77 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:11:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:10,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:11:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:10,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:11:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960730292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960730292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960730292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960730292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:10,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/e3d5e17f9ec84c6d96debed5c35ef9a5 2024-11-18T20:11:10,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/2a43f55102fd4f89b6cde1b11617fdc2 is 50, key is test_row_0/C:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:10,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742438_1614 (size=12301) 2024-11-18T20:11:10,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/2a43f55102fd4f89b6cde1b11617fdc2 2024-11-18T20:11:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/99813ef05a7448e9aaf4f59b7d4d77b1 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1 2024-11-18T20:11:10,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1, entries=150, sequenceid=282, filesize=30.5 K 2024-11-18T20:11:10,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/e3d5e17f9ec84c6d96debed5c35ef9a5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e3d5e17f9ec84c6d96debed5c35ef9a5 2024-11-18T20:11:10,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e3d5e17f9ec84c6d96debed5c35ef9a5, entries=150, sequenceid=282, filesize=12.0 K 2024-11-18T20:11:10,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/2a43f55102fd4f89b6cde1b11617fdc2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/2a43f55102fd4f89b6cde1b11617fdc2 2024-11-18T20:11:10,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/2a43f55102fd4f89b6cde1b11617fdc2, entries=150, sequenceid=282, filesize=12.0 K 2024-11-18T20:11:10,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for fc81ed96fcb7834182497904e66e6d77 in 1300ms, sequenceid=282, compaction requested=true 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:10,404 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:10,404 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:11:10,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:11:10,408 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,408 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,408 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c3a43027969f433ea47634a4cfa6d306, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=91.8 K 2024-11-18T20:11:10,408 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/dcb536f5cd7b4a349488b401eef856a4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/cafe774bbf9549fa95e5e674b1286aec, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e3d5e17f9ec84c6d96debed5c35ef9a5] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=36.3 K 2024-11-18T20:11:10,408 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c3a43027969f433ea47634a4cfa6d306, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1] 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb536f5cd7b4a349488b401eef856a4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960664597 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a43027969f433ea47634a4cfa6d306, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960664597 2024-11-18T20:11:10,408 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting cafe774bbf9549fa95e5e674b1286aec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960665785 2024-11-18T20:11:10,409 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d5e17f9ec84c6d96debed5c35ef9a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731960667962 2024-11-18T20:11:10,409 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e1e633eddc479ca994261d4d23517c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960665785 2024-11-18T20:11:10,409 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99813ef05a7448e9aaf4f59b7d4d77b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731960667962 2024-11-18T20:11:10,414 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:10,414 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:10,415 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/5e9b70d4b1154b07bbfb2ffc37a1905a is 50, key is test_row_0/B:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:10,416 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111804dd595577844dd4a88317460f001c4e_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:10,419 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111804dd595577844dd4a88317460f001c4e_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:10,419 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111804dd595577844dd4a88317460f001c4e_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:10,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:11:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,424 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:11:10,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742439_1615 (size=12949) 2024-11-18T20:11:10,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742440_1616 (size=4469) 2024-11-18T20:11:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b50ebd021b143349568aeafe9cf1ded_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960669150/Put/seqid=0 2024-11-18T20:11:10,446 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#520 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:10,447 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/584b311da03446e280f35d432502805d is 175, key is test_row_0/A:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:10,457 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/5e9b70d4b1154b07bbfb2ffc37a1905a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/5e9b70d4b1154b07bbfb2ffc37a1905a 2024-11-18T20:11:10,463 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into 5e9b70d4b1154b07bbfb2ffc37a1905a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:10,463 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:10,463 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=13, startTime=1731960670404; duration=0sec 2024-11-18T20:11:10,463 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:10,464 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:11:10,464 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:10,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742441_1617 (size=12454) 2024-11-18T20:11:10,465 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:10,465 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:11:10,465 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:10,465 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/a62770871eb542899fa4e24bcbac0376, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/38bc89d1988c4dca9892c6d4ff85c4b2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/2a43f55102fd4f89b6cde1b11617fdc2] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=36.3 K 2024-11-18T20:11:10,465 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting a62770871eb542899fa4e24bcbac0376, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960664597 2024-11-18T20:11:10,466 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bc89d1988c4dca9892c6d4ff85c4b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960665785 2024-11-18T20:11:10,466 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a43f55102fd4f89b6cde1b11617fdc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731960667962 2024-11-18T20:11:10,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742442_1618 (size=31903) 2024-11-18T20:11:10,472 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:10,473 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/6faf05d4498b491db3d48609fffdc490 is 50, key is test_row_0/C:col10/1731960669103/Put/seqid=0 2024-11-18T20:11:10,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742443_1619 (size=12949) 2024-11-18T20:11:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:10,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:10,887 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b50ebd021b143349568aeafe9cf1ded_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b50ebd021b143349568aeafe9cf1ded_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:10,887 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/6faf05d4498b491db3d48609fffdc490 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6faf05d4498b491db3d48609fffdc490 2024-11-18T20:11:10,887 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/584b311da03446e280f35d432502805d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/584b311da03446e280f35d432502805d 2024-11-18T20:11:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/758256463249454b855515e0e7980d22, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/758256463249454b855515e0e7980d22 is 175, key is test_row_0/A:col10/1731960669150/Put/seqid=0 2024-11-18T20:11:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742444_1620 (size=31255) 2024-11-18T20:11:10,900 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into 584b311da03446e280f35d432502805d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:10,900 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into 6faf05d4498b491db3d48609fffdc490(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:10,900 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:10,900 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:10,900 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=13, startTime=1731960670404; duration=0sec 2024-11-18T20:11:10,900 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=13, startTime=1731960670404; duration=0sec 2024-11-18T20:11:10,900 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:10,900 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:10,900 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:11:10,900 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:11:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:11,293 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/758256463249454b855515e0e7980d22 2024-11-18T20:11:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/23f4cf69fba64c4f92eb45d2391842d2 is 50, key is test_row_0/B:col10/1731960669150/Put/seqid=0 2024-11-18T20:11:11,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742445_1621 (size=12301) 2024-11-18T20:11:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:11,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. as already flushing 2024-11-18T20:11:11,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960731323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960731326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960731326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960731327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960731432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960731432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960731432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960731441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960731642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960731642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960731642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960731651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/23f4cf69fba64c4f92eb45d2391842d2 2024-11-18T20:11:11,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/c8c38aabbcb244fa97076d95fc1b0d48 is 50, key is test_row_0/C:col10/1731960669150/Put/seqid=0 2024-11-18T20:11:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742446_1622 (size=12301) 2024-11-18T20:11:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960731952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960731952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960731952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960731962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:12,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/c8c38aabbcb244fa97076d95fc1b0d48 2024-11-18T20:11:12,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/758256463249454b855515e0e7980d22 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22 2024-11-18T20:11:12,126 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22, entries=150, sequenceid=295, filesize=30.5 K 2024-11-18T20:11:12,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/23f4cf69fba64c4f92eb45d2391842d2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/23f4cf69fba64c4f92eb45d2391842d2 2024-11-18T20:11:12,130 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/23f4cf69fba64c4f92eb45d2391842d2, entries=150, sequenceid=295, filesize=12.0 K 2024-11-18T20:11:12,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/c8c38aabbcb244fa97076d95fc1b0d48 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/c8c38aabbcb244fa97076d95fc1b0d48 2024-11-18T20:11:12,134 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/c8c38aabbcb244fa97076d95fc1b0d48, entries=150, sequenceid=295, filesize=12.0 K 2024-11-18T20:11:12,135 INFO [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fc81ed96fcb7834182497904e66e6d77 in 1711ms, sequenceid=295, compaction requested=false 2024-11-18T20:11:12,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:12,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:12,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d8f95bb18b58:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-18T20:11:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-18T20:11:12,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-18T20:11:12,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-18T20:11:12,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1770 sec 2024-11-18T20:11:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:12,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:11:12,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:11:12,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111883d8d0e423484de0bf77dc9b4317257c_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960732473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960732474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960732475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960732475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742447_1623 (size=17534) 2024-11-18T20:11:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960732581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960732581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960732581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960732582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54840 deadline: 1731960732792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54794 deadline: 1731960732792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54842 deadline: 1731960732792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:11:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1731960732792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:11:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., hostname=d8f95bb18b58,42447,1731960497538, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:11:12,884 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:12,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111883d8d0e423484de0bf77dc9b4317257c_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111883d8d0e423484de0bf77dc9b4317257c_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:12,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/fcbf537e105f4460bad7d74e9b57fa7a, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:12,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/fcbf537e105f4460bad7d74e9b57fa7a is 175, key is test_row_0/A:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:12,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742448_1624 (size=48639) 2024-11-18T20:11:12,895 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/fcbf537e105f4460bad7d74e9b57fa7a 2024-11-18T20:11:12,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/3fb4b3b5fb3d46109a85dde9e32372d9 is 50, key is test_row_0/B:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742449_1625 (size=12301) 2024-11-18T20:11:12,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/3fb4b3b5fb3d46109a85dde9e32372d9 2024-11-18T20:11:12,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/657eb5347cf640b6915f87deb1da3029 is 50, key is test_row_0/C:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:12,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742450_1626 (size=12301) 2024-11-18T20:11:12,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/657eb5347cf640b6915f87deb1da3029 2024-11-18T20:11:12,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/fcbf537e105f4460bad7d74e9b57fa7a as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a 2024-11-18T20:11:12,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a, entries=250, sequenceid=324, filesize=47.5 K 2024-11-18T20:11:12,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/3fb4b3b5fb3d46109a85dde9e32372d9 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3fb4b3b5fb3d46109a85dde9e32372d9 2024-11-18T20:11:12,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3fb4b3b5fb3d46109a85dde9e32372d9, entries=150, sequenceid=324, filesize=12.0 K 2024-11-18T20:11:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/657eb5347cf640b6915f87deb1da3029 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/657eb5347cf640b6915f87deb1da3029 2024-11-18T20:11:13,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/657eb5347cf640b6915f87deb1da3029, entries=150, sequenceid=324, filesize=12.0 K 2024-11-18T20:11:13,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fc81ed96fcb7834182497904e66e6d77 in 560ms, sequenceid=324, compaction requested=true 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc81ed96fcb7834182497904e66e6d77:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:11:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T20:11:13,026 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:13,027 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:13,029 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:13,029 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/A is initiating minor compaction (all files) 2024-11-18T20:11:13,029 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/A in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:13,029 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/584b311da03446e280f35d432502805d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=109.2 K 2024-11-18T20:11:13,029 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:13,029 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. files: [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/584b311da03446e280f35d432502805d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a] 2024-11-18T20:11:13,029 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584b311da03446e280f35d432502805d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731960667962 2024-11-18T20:11:13,030 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 758256463249454b855515e0e7980d22, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731960669104 2024-11-18T20:11:13,030 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcbf537e105f4460bad7d74e9b57fa7a, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731960671325 2024-11-18T20:11:13,047 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:13,047 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/C is initiating minor compaction (all files) 2024-11-18T20:11:13,047 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/C in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:13,047 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6faf05d4498b491db3d48609fffdc490, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/c8c38aabbcb244fa97076d95fc1b0d48, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/657eb5347cf640b6915f87deb1da3029] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=36.7 K 2024-11-18T20:11:13,055 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6faf05d4498b491db3d48609fffdc490, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731960667962 2024-11-18T20:11:13,059 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c38aabbcb244fa97076d95fc1b0d48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731960669104 2024-11-18T20:11:13,063 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 657eb5347cf640b6915f87deb1da3029, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731960671325 2024-11-18T20:11:13,069 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:13,090 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118b87aefbbf7c54e7c9ca6378284d668d1_fc81ed96fcb7834182497904e66e6d77 store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:13,092 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118b87aefbbf7c54e7c9ca6378284d668d1_fc81ed96fcb7834182497904e66e6d77, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:13,092 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b87aefbbf7c54e7c9ca6378284d668d1_fc81ed96fcb7834182497904e66e6d77 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:11:13,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:11:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42447 {}] regionserver.HRegion(8855): Flush requested on fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:13,116 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:13,116 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/413188c51ef4467e86f729597a005191 is 50, key is test_row_0/C:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:13,124 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-11-18T20:11:13,124 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,124 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,124 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,128 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-11-18T20:11:13,128 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,128 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,128 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742451_1627 (size=4469) 2024-11-18T20:11:13,135 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-18T20:11:13,135 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,135 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,135 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,136 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-11-18T20:11:13,136 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,136 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,136 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,137 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-11-18T20:11:13,137 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,137 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-11-18T20:11:13,137 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,137 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,137 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,137 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,137 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742452_1628 (size=13051) 2024-11-18T20:11:13,143 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-11-18T20:11:13,143 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,143 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,144 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,144 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-11-18T20:11:13,144 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,144 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,144 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118aa44c658d4f649f28146b7b4af51f2f4_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960673105/Put/seqid=0 2024-11-18T20:11:13,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,148 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-18T20:11:13,148 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:13,148 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,148 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:13,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:13,160 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/413188c51ef4467e86f729597a005191 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/413188c51ef4467e86f729597a005191 2024-11-18T20:11:13,180 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/C of fc81ed96fcb7834182497904e66e6d77 into 413188c51ef4467e86f729597a005191(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:13,180 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:13,180 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/C, priority=13, startTime=1731960673024; duration=0sec 2024-11-18T20:11:13,180 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:11:13,180 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:C 2024-11-18T20:11:13,180 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:11:13,188 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:11:13,188 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1541): fc81ed96fcb7834182497904e66e6d77/B is initiating minor compaction (all files) 2024-11-18T20:11:13,188 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc81ed96fcb7834182497904e66e6d77/B in TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:13,188 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/5e9b70d4b1154b07bbfb2ffc37a1905a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/23f4cf69fba64c4f92eb45d2391842d2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3fb4b3b5fb3d46109a85dde9e32372d9] into tmpdir=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp, totalSize=36.7 K 2024-11-18T20:11:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742453_1629 (size=12454) 2024-11-18T20:11:13,190 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9b70d4b1154b07bbfb2ffc37a1905a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731960667962 2024-11-18T20:11:13,191 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f4cf69fba64c4f92eb45d2391842d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731960669104 2024-11-18T20:11:13,191 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb4b3b5fb3d46109a85dde9e32372d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731960671325 2024-11-18T20:11:13,201 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:13,201 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/57643917c76041c084749dda2aaae15d is 50, key is test_row_0/B:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:13,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742454_1630 (size=13051) 2024-11-18T20:11:13,533 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc81ed96fcb7834182497904e66e6d77#A#compaction#528 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:11:13,533 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/42c761c397194fb2b6081f1212afe263 is 175, key is test_row_0/A:col10/1731960672464/Put/seqid=0 2024-11-18T20:11:13,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742455_1631 (size=32005) 2024-11-18T20:11:13,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:13,593 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118aa44c658d4f649f28146b7b4af51f2f4_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118aa44c658d4f649f28146b7b4af51f2f4_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:13,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7bbd53136ff648c3be53a564f6a36b04, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:13,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7bbd53136ff648c3be53a564f6a36b04 is 175, key is test_row_0/A:col10/1731960673105/Put/seqid=0 2024-11-18T20:11:13,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742456_1632 (size=31255) 2024-11-18T20:11:13,617 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/57643917c76041c084749dda2aaae15d as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/57643917c76041c084749dda2aaae15d 2024-11-18T20:11:13,621 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/B of fc81ed96fcb7834182497904e66e6d77 into 57643917c76041c084749dda2aaae15d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:13,621 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:13,621 INFO [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/B, priority=13, startTime=1731960673024; duration=0sec 2024-11-18T20:11:13,621 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:13,621 DEBUG [RS:0;d8f95bb18b58:42447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:B 2024-11-18T20:11:13,948 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/42c761c397194fb2b6081f1212afe263 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/42c761c397194fb2b6081f1212afe263 2024-11-18T20:11:13,956 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc81ed96fcb7834182497904e66e6d77/A of fc81ed96fcb7834182497904e66e6d77 into 42c761c397194fb2b6081f1212afe263(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:11:13,956 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:13,956 INFO [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., storeName=fc81ed96fcb7834182497904e66e6d77/A, priority=13, startTime=1731960673024; duration=0sec 2024-11-18T20:11:13,956 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:11:13,956 DEBUG [RS:0;d8f95bb18b58:42447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc81ed96fcb7834182497904e66e6d77:A 2024-11-18T20:11:13,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7bbd53136ff648c3be53a564f6a36b04 2024-11-18T20:11:14,003 INFO [Thread-2401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2401. 2024-11-18T20:11:14,003 DEBUG [Thread-2401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:11:14,003 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:14,003 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:14,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:14,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/7505e097685746e8a50404a1428939ae is 50, key is test_row_0/B:col10/1731960673105/Put/seqid=0 2024-11-18T20:11:14,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742457_1633 (size=12301) 2024-11-18T20:11:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:11:14,132 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3065 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3170 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3017 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3037 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3157 2024-11-18T20:11:14,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:11:14,132 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:11:14,133 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:11:14,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:14,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:14,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:14,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:11:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:11:14,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960674137"}]},"ts":"1731960674137"} 2024-11-18T20:11:14,139 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:11:14,139 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:11:14,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:11:14,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, UNASSIGN}] 2024-11-18T20:11:14,142 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, UNASSIGN 2024-11-18T20:11:14,143 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=CLOSING, regionLocation=d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:14,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, UNASSIGN because future has completed 2024-11-18T20:11:14,145 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:11:14,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538}] 2024-11-18T20:11:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:11:14,298 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:14,298 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:11:14,298 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing fc81ed96fcb7834182497904e66e6d77, disabling compactions & flushes 2024-11-18T20:11:14,299 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:14,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/7505e097685746e8a50404a1428939ae 2024-11-18T20:11:14,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/f5ad1611cd9d4ff399a06c659c91c8db is 50, key is test_row_0/C:col10/1731960673105/Put/seqid=0 2024-11-18T20:11:14,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742458_1634 (size=12301) 2024-11-18T20:11:14,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/f5ad1611cd9d4ff399a06c659c91c8db 2024-11-18T20:11:14,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/7bbd53136ff648c3be53a564f6a36b04 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7bbd53136ff648c3be53a564f6a36b04 2024-11-18T20:11:14,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7bbd53136ff648c3be53a564f6a36b04, entries=150, sequenceid=335, filesize=30.5 K 2024-11-18T20:11:14,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/7505e097685746e8a50404a1428939ae as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/7505e097685746e8a50404a1428939ae 2024-11-18T20:11:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:11:14,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/7505e097685746e8a50404a1428939ae, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T20:11:14,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/f5ad1611cd9d4ff399a06c659c91c8db as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/f5ad1611cd9d4ff399a06c659c91c8db 2024-11-18T20:11:14,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/f5ad1611cd9d4ff399a06c659c91c8db, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T20:11:14,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=60.38 KB/61830 for fc81ed96fcb7834182497904e66e6d77 in 1357ms, sequenceid=335, compaction requested=false 2024-11-18T20:11:14,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc81ed96fcb7834182497904e66e6d77: 2024-11-18T20:11:14,464 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:14,464 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:14,464 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. after waiting 0 ms 2024-11-18T20:11:14,464 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:14,464 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing fc81ed96fcb7834182497904e66e6d77 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:11:14,469 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111809f7291d47d9428cae3c944652700a8f_fc81ed96fcb7834182497904e66e6d77 is 50, key is test_row_0/A:col10/1731960674002/Put/seqid=0 2024-11-18T20:11:14,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742459_1635 (size=12454) 2024-11-18T20:11:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:11:14,873 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:11:14,876 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111809f7291d47d9428cae3c944652700a8f_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111809f7291d47d9428cae3c944652700a8f_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:14,877 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a2a880914c34456499b2979636f495e5, store: [table=TestAcidGuarantees family=A region=fc81ed96fcb7834182497904e66e6d77] 2024-11-18T20:11:14,877 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a2a880914c34456499b2979636f495e5 is 175, key is test_row_0/A:col10/1731960674002/Put/seqid=0 2024-11-18T20:11:14,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742460_1636 (size=31255) 2024-11-18T20:11:14,881 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a2a880914c34456499b2979636f495e5 2024-11-18T20:11:14,887 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/0b2461f2c69644e18c1abdc3652a65d2 is 50, key is test_row_0/B:col10/1731960674002/Put/seqid=0 2024-11-18T20:11:14,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742461_1637 (size=12301) 2024-11-18T20:11:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:11:15,290 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/0b2461f2c69644e18c1abdc3652a65d2 2024-11-18T20:11:15,297 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/02c71d52d871456183701483ae9a37d8 is 50, key is test_row_0/C:col10/1731960674002/Put/seqid=0 2024-11-18T20:11:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742462_1638 (size=12301) 2024-11-18T20:11:15,712 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/02c71d52d871456183701483ae9a37d8 2024-11-18T20:11:15,716 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/A/a2a880914c34456499b2979636f495e5 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a2a880914c34456499b2979636f495e5 2024-11-18T20:11:15,719 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a2a880914c34456499b2979636f495e5, entries=150, sequenceid=350, filesize=30.5 K 2024-11-18T20:11:15,719 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/B/0b2461f2c69644e18c1abdc3652a65d2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/0b2461f2c69644e18c1abdc3652a65d2 2024-11-18T20:11:15,722 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/0b2461f2c69644e18c1abdc3652a65d2, entries=150, sequenceid=350, filesize=12.0 K 2024-11-18T20:11:15,723 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/.tmp/C/02c71d52d871456183701483ae9a37d8 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/02c71d52d871456183701483ae9a37d8 2024-11-18T20:11:15,726 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/02c71d52d871456183701483ae9a37d8, entries=150, sequenceid=350, filesize=12.0 K 2024-11-18T20:11:15,727 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for fc81ed96fcb7834182497904e66e6d77 in 1263ms, sequenceid=350, compaction requested=true 2024-11-18T20:11:15,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/24dd0284b6a643f494f2ac00506bd19c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/0302c19a2aae48aabb02eb4b6ce253e6, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/de33e6b031a44668b1b8a6a47395769e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ef28a5dc5ab7492ba5eda390be530362, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c3a43027969f433ea47634a4cfa6d306, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/584b311da03446e280f35d432502805d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a] to archive 2024-11-18T20:11:15,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:11:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7cd4be5bb0c74e6e8ed5d5b4968557f8 2024-11-18T20:11:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/290c05dc7d6640f2882ad6455427cf0d 2024-11-18T20:11:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/6ccec64cd3ad4559826b62fde79e660a 2024-11-18T20:11:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/24dd0284b6a643f494f2ac00506bd19c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/24dd0284b6a643f494f2ac00506bd19c 2024-11-18T20:11:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/4fbc1899fe1f4d56a24668ba8348d088 2024-11-18T20:11:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a16ad30071854dc69bbd6863e29f4152 2024-11-18T20:11:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/0302c19a2aae48aabb02eb4b6ce253e6 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/0302c19a2aae48aabb02eb4b6ce253e6 2024-11-18T20:11:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/25746ed8a8d74639a2c033e8623b2beb 2024-11-18T20:11:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/de33e6b031a44668b1b8a6a47395769e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/de33e6b031a44668b1b8a6a47395769e 2024-11-18T20:11:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/03be40cdaf31466ba0657cc7d46eabd4 2024-11-18T20:11:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ae08defe3baa41efb9b41fecbfbafe70 2024-11-18T20:11:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ef28a5dc5ab7492ba5eda390be530362 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/ef28a5dc5ab7492ba5eda390be530362 2024-11-18T20:11:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e91dab29a7c34b7b850b50eabb675a68 2024-11-18T20:11:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c953dc9575514e68864877b332dc4d63 2024-11-18T20:11:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/e0e0bdd48dad4f519ec2d80161cd275b 2024-11-18T20:11:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/50b34be46a9842b285232fb548468ca1 2024-11-18T20:11:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c3a43027969f433ea47634a4cfa6d306 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/c3a43027969f433ea47634a4cfa6d306 2024-11-18T20:11:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/f3e1e633eddc479ca994261d4d23517c 2024-11-18T20:11:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/584b311da03446e280f35d432502805d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/584b311da03446e280f35d432502805d 2024-11-18T20:11:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/99813ef05a7448e9aaf4f59b7d4d77b1 2024-11-18T20:11:15,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/758256463249454b855515e0e7980d22 2024-11-18T20:11:15,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/fcbf537e105f4460bad7d74e9b57fa7a 2024-11-18T20:11:15,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/51e32d5693234020909479353501691c, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d92055caf7f940f6a6836083bed78d6f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/8e477f74ed2743f78904fc6e6bd111fc, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/2d63cacbc55245dfb465e2dd70533ff2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/a3176e5fe10c4bc6a7a5072a9e97cd9e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d0cd8f5fbc5d4bedbaf393ff20530f9e, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e425ac4877354635a1a5b02d43bea44f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/642b0675cd2d440580dcb02d8dcf70e3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/69c6371a826744779db02b71d637ff1d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/321b04be93df4a038b7d0a484c90f223, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/52fee884b8c346589c424245718be3cb, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/9a24b680c01f4a77bc9d1040cee7744a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d123b579bd3c416f8a9a5c00c50a76f1, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d3d947e225f84867bf2bb1c3f9cf824b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3c46ff30bdd547eda4f4c3b4ccf55963, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/dcb536f5cd7b4a349488b401eef856a4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/29c79633f4c34f9e8aca2d51a714f553, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/cafe774bbf9549fa95e5e674b1286aec, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/5e9b70d4b1154b07bbfb2ffc37a1905a, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e3d5e17f9ec84c6d96debed5c35ef9a5, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/23f4cf69fba64c4f92eb45d2391842d2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3fb4b3b5fb3d46109a85dde9e32372d9] to archive 2024-11-18T20:11:15,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:11:15,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/51e32d5693234020909479353501691c to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/51e32d5693234020909479353501691c 2024-11-18T20:11:15,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d92055caf7f940f6a6836083bed78d6f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d92055caf7f940f6a6836083bed78d6f 2024-11-18T20:11:15,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/8e477f74ed2743f78904fc6e6bd111fc to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/8e477f74ed2743f78904fc6e6bd111fc 2024-11-18T20:11:15,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/2d63cacbc55245dfb465e2dd70533ff2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/2d63cacbc55245dfb465e2dd70533ff2 2024-11-18T20:11:15,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/a3176e5fe10c4bc6a7a5072a9e97cd9e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/a3176e5fe10c4bc6a7a5072a9e97cd9e 2024-11-18T20:11:15,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d0cd8f5fbc5d4bedbaf393ff20530f9e to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d0cd8f5fbc5d4bedbaf393ff20530f9e 2024-11-18T20:11:15,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e425ac4877354635a1a5b02d43bea44f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e425ac4877354635a1a5b02d43bea44f 2024-11-18T20:11:15,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/642b0675cd2d440580dcb02d8dcf70e3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/642b0675cd2d440580dcb02d8dcf70e3 2024-11-18T20:11:15,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/69c6371a826744779db02b71d637ff1d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/69c6371a826744779db02b71d637ff1d 2024-11-18T20:11:15,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/321b04be93df4a038b7d0a484c90f223 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/321b04be93df4a038b7d0a484c90f223 2024-11-18T20:11:15,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/52fee884b8c346589c424245718be3cb to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/52fee884b8c346589c424245718be3cb 2024-11-18T20:11:15,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/9a24b680c01f4a77bc9d1040cee7744a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/9a24b680c01f4a77bc9d1040cee7744a 2024-11-18T20:11:15,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d123b579bd3c416f8a9a5c00c50a76f1 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d123b579bd3c416f8a9a5c00c50a76f1 2024-11-18T20:11:15,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d3d947e225f84867bf2bb1c3f9cf824b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/d3d947e225f84867bf2bb1c3f9cf824b 2024-11-18T20:11:15,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3c46ff30bdd547eda4f4c3b4ccf55963 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3c46ff30bdd547eda4f4c3b4ccf55963 2024-11-18T20:11:15,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/dcb536f5cd7b4a349488b401eef856a4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/dcb536f5cd7b4a349488b401eef856a4 2024-11-18T20:11:15,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/29c79633f4c34f9e8aca2d51a714f553 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/29c79633f4c34f9e8aca2d51a714f553 2024-11-18T20:11:15,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/cafe774bbf9549fa95e5e674b1286aec to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/cafe774bbf9549fa95e5e674b1286aec 2024-11-18T20:11:15,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/5e9b70d4b1154b07bbfb2ffc37a1905a to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/5e9b70d4b1154b07bbfb2ffc37a1905a 2024-11-18T20:11:15,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e3d5e17f9ec84c6d96debed5c35ef9a5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/e3d5e17f9ec84c6d96debed5c35ef9a5 2024-11-18T20:11:15,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/23f4cf69fba64c4f92eb45d2391842d2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/23f4cf69fba64c4f92eb45d2391842d2 2024-11-18T20:11:15,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3fb4b3b5fb3d46109a85dde9e32372d9 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/3fb4b3b5fb3d46109a85dde9e32372d9 2024-11-18T20:11:15,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/1ea4b61d72f04632a8468e168a2a4a73, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/8f16feb72e124b4789acc8aa194ea303, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/05265071b26e4e9b8f2de373fb1edcb3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/0dce9dfcd2a5492298da57440332cb8b, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4275e3eb146e400b98fc3445dfa4f0b3, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6afac72c98a94f328fb108adacf72e21, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/901b730a779e4f818463352944738ea4, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4e93b7a8a5114e9fbfa6cfc18603e1cd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/79381ecf91834f13b4e353f73820ff85, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/30e1304212404c6cbda5c928490dddbd, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ecc8a05265164487be7da9c2aab79c49, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/b558fe2fbf674445bd234343855db8f2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/af21f837501548efb36ee3351ad8a281, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/9fdd8afb6200490e90a42ecc65a5986f, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ad177fb6e5164c94bab0461488c5a50d, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/a62770871eb542899fa4e24bcbac0376, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/5a719077f8454b058e3d9c4fc35c3ef0, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/38bc89d1988c4dca9892c6d4ff85c4b2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6faf05d4498b491db3d48609fffdc490, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/2a43f55102fd4f89b6cde1b11617fdc2, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/c8c38aabbcb244fa97076d95fc1b0d48, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/657eb5347cf640b6915f87deb1da3029] to archive 2024-11-18T20:11:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:11:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/1ea4b61d72f04632a8468e168a2a4a73 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/1ea4b61d72f04632a8468e168a2a4a73 2024-11-18T20:11:15,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/8f16feb72e124b4789acc8aa194ea303 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/8f16feb72e124b4789acc8aa194ea303 2024-11-18T20:11:15,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/05265071b26e4e9b8f2de373fb1edcb3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/05265071b26e4e9b8f2de373fb1edcb3 2024-11-18T20:11:15,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/0dce9dfcd2a5492298da57440332cb8b to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/0dce9dfcd2a5492298da57440332cb8b 2024-11-18T20:11:15,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4275e3eb146e400b98fc3445dfa4f0b3 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4275e3eb146e400b98fc3445dfa4f0b3 2024-11-18T20:11:15,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6afac72c98a94f328fb108adacf72e21 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6afac72c98a94f328fb108adacf72e21 2024-11-18T20:11:15,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/901b730a779e4f818463352944738ea4 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/901b730a779e4f818463352944738ea4 2024-11-18T20:11:15,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4e93b7a8a5114e9fbfa6cfc18603e1cd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/4e93b7a8a5114e9fbfa6cfc18603e1cd 2024-11-18T20:11:15,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/79381ecf91834f13b4e353f73820ff85 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/79381ecf91834f13b4e353f73820ff85 2024-11-18T20:11:15,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/30e1304212404c6cbda5c928490dddbd to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/30e1304212404c6cbda5c928490dddbd 2024-11-18T20:11:15,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ecc8a05265164487be7da9c2aab79c49 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ecc8a05265164487be7da9c2aab79c49 2024-11-18T20:11:15,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/b558fe2fbf674445bd234343855db8f2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/b558fe2fbf674445bd234343855db8f2 2024-11-18T20:11:15,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/af21f837501548efb36ee3351ad8a281 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/af21f837501548efb36ee3351ad8a281 2024-11-18T20:11:15,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/9fdd8afb6200490e90a42ecc65a5986f to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/9fdd8afb6200490e90a42ecc65a5986f 2024-11-18T20:11:15,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ad177fb6e5164c94bab0461488c5a50d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/ad177fb6e5164c94bab0461488c5a50d 2024-11-18T20:11:15,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/a62770871eb542899fa4e24bcbac0376 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/a62770871eb542899fa4e24bcbac0376 2024-11-18T20:11:15,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/5a719077f8454b058e3d9c4fc35c3ef0 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/5a719077f8454b058e3d9c4fc35c3ef0 2024-11-18T20:11:15,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/38bc89d1988c4dca9892c6d4ff85c4b2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/38bc89d1988c4dca9892c6d4ff85c4b2 2024-11-18T20:11:15,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6faf05d4498b491db3d48609fffdc490 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/6faf05d4498b491db3d48609fffdc490 2024-11-18T20:11:15,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/2a43f55102fd4f89b6cde1b11617fdc2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/2a43f55102fd4f89b6cde1b11617fdc2 2024-11-18T20:11:15,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/c8c38aabbcb244fa97076d95fc1b0d48 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/c8c38aabbcb244fa97076d95fc1b0d48 2024-11-18T20:11:15,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/657eb5347cf640b6915f87deb1da3029 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/657eb5347cf640b6915f87deb1da3029 2024-11-18T20:11:15,808 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=4 2024-11-18T20:11:15,808 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. 2024-11-18T20:11:15,808 DEBUG [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for fc81ed96fcb7834182497904e66e6d77: Waiting for close lock at 1731960674298Running coprocessor pre-close hooks at 1731960674298Disabling compacts and flushes for region at 1731960674298Disabling writes for close at 1731960674464 (+166 ms)Obtaining lock to block concurrent updates at 1731960674464Preparing flush snapshotting stores in fc81ed96fcb7834182497904e66e6d77 at 1731960674464Finished memstore snapshotting TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77., syncing WAL and waiting on mvcc, flushsize=dataSize=61830, getHeapSize=162720, getOffHeapSize=0, getCellsCount=1350 at 1731960674464Flushing stores of TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77. at 1731960674464Flushing fc81ed96fcb7834182497904e66e6d77/A: creating writer at 1731960674464Flushing mob file fc81ed96fcb7834182497904e66e6d77/A: appending metadata at 1731960674469 (+5 ms)Flushing mob file fc81ed96fcb7834182497904e66e6d77/A: closing flushed file at 1731960674469Flushing fc81ed96fcb7834182497904e66e6d77/A: appending metadata at 1731960674877 (+408 ms)Flushing fc81ed96fcb7834182497904e66e6d77/A: closing flushed file at 1731960674877Flushing fc81ed96fcb7834182497904e66e6d77/B: creating writer at 1731960674884 (+7 ms)Flushing fc81ed96fcb7834182497904e66e6d77/B: appending metadata at 1731960674886 (+2 ms)Flushing fc81ed96fcb7834182497904e66e6d77/B: closing flushed file at 1731960674886Flushing fc81ed96fcb7834182497904e66e6d77/C: creating writer at 1731960675294 (+408 ms)Flushing fc81ed96fcb7834182497904e66e6d77/C: appending metadata at 1731960675296 (+2 ms)Flushing fc81ed96fcb7834182497904e66e6d77/C: closing flushed file at 1731960675297 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@578594e1: reopening flushed file at 1731960675715 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8186b9c: reopening flushed file at 1731960675719 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53ba5d57: reopening flushed file at 1731960675723 (+4 ms)Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for fc81ed96fcb7834182497904e66e6d77 in 1263ms, sequenceid=350, compaction requested=true at 1731960675727 (+4 ms)Writing region close event to WAL at 1731960675804 (+77 ms)Running coprocessor post-close hooks at 1731960675808 (+4 ms)Closed at 1731960675808 2024-11-18T20:11:15,810 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=fc81ed96fcb7834182497904e66e6d77, regionState=CLOSED 2024-11-18T20:11:15,811 INFO [RS_CLOSE_REGION-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:15,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 because future has completed 2024-11-18T20:11:15,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-18T20:11:15,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure fc81ed96fcb7834182497904e66e6d77, server=d8f95bb18b58,42447,1731960497538 in 1.6670 sec 2024-11-18T20:11:15,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-18T20:11:15,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc81ed96fcb7834182497904e66e6d77, UNASSIGN in 1.6730 sec 2024-11-18T20:11:15,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-18T20:11:15,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6770 sec 2024-11-18T20:11:15,820 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960675819"}]},"ts":"1731960675819"} 2024-11-18T20:11:15,821 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:11:15,821 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:11:15,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6880 sec 2024-11-18T20:11:15,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:11:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:11:16,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:11:16,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:11:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:16,284 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:16,285 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:11:16,287 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C, FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/recovered.edits] 2024-11-18T20:11:16,290 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/42c761c397194fb2b6081f1212afe263 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/42c761c397194fb2b6081f1212afe263 2024-11-18T20:11:16,291 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7bbd53136ff648c3be53a564f6a36b04 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/7bbd53136ff648c3be53a564f6a36b04 2024-11-18T20:11:16,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a2a880914c34456499b2979636f495e5 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/A/a2a880914c34456499b2979636f495e5 2024-11-18T20:11:16,293 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/0b2461f2c69644e18c1abdc3652a65d2 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/0b2461f2c69644e18c1abdc3652a65d2 2024-11-18T20:11:16,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/57643917c76041c084749dda2aaae15d to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/57643917c76041c084749dda2aaae15d 2024-11-18T20:11:16,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/7505e097685746e8a50404a1428939ae to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/B/7505e097685746e8a50404a1428939ae 2024-11-18T20:11:16,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/02c71d52d871456183701483ae9a37d8 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/02c71d52d871456183701483ae9a37d8 2024-11-18T20:11:16,297 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/413188c51ef4467e86f729597a005191 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/413188c51ef4467e86f729597a005191 2024-11-18T20:11:16,298 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/f5ad1611cd9d4ff399a06c659c91c8db to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/C/f5ad1611cd9d4ff399a06c659c91c8db 2024-11-18T20:11:16,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/recovered.edits/353.seqid to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77/recovered.edits/353.seqid 2024-11-18T20:11:16,300 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/default/TestAcidGuarantees/fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,300 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:11:16,300 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:11:16,301 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:11:16,303 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111804db54f8b0fa4bfd9b2ea673714974ca_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111804db54f8b0fa4bfd9b2ea673714974ca_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,304 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111809f7291d47d9428cae3c944652700a8f_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111809f7291d47d9428cae3c944652700a8f_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,306 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a01e0d16d4c40d29c2a79571a13c5a1_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a01e0d16d4c40d29c2a79571a13c5a1_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,307 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181be119c152f84704835019555c270233_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181be119c152f84704835019555c270233_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,308 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111833285baba3b14604a0930aa08136abee_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111833285baba3b14604a0930aa08136abee_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,309 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183b85214f123243fab7e17f6815d98e9d_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183b85214f123243fab7e17f6815d98e9d_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,311 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843f8fcc02cd449c198575bb4288d4410_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843f8fcc02cd449c198575bb4288d4410_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,312 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111847df8a220e104b6d8fc00c9cdd0bdeaf_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111847df8a220e104b6d8fc00c9cdd0bdeaf_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,313 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185e22b318f2eb46b3a433f316426a7287_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185e22b318f2eb46b3a433f316426a7287_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,314 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186c457adffd0440b59f00ead62412b85f_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186c457adffd0440b59f00ead62412b85f_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,315 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b50ebd021b143349568aeafe9cf1ded_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b50ebd021b143349568aeafe9cf1ded_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,316 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118829dba8b58694d86b45e9714b31cb380_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118829dba8b58694d86b45e9714b31cb380_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,317 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111883d8d0e423484de0bf77dc9b4317257c_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111883d8d0e423484de0bf77dc9b4317257c_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,318 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188aa4adcc41cb4b79bf7a7a17168e4639_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188aa4adcc41cb4b79bf7a7a17168e4639_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,318 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118aa44c658d4f649f28146b7b4af51f2f4_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118aa44c658d4f649f28146b7b4af51f2f4_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,319 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b8d0b371d80341f694d612c3a5990bd2_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b8d0b371d80341f694d612c3a5990bd2_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,320 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d6d77590a209474580eb76d4232080ef_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d6d77590a209474580eb76d4232080ef_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,321 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f246c13c74aa44cbb84de61b90d318c2_fc81ed96fcb7834182497904e66e6d77 to hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f246c13c74aa44cbb84de61b90d318c2_fc81ed96fcb7834182497904e66e6d77 2024-11-18T20:11:16,322 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:11:16,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:16,326 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:11:16,327 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:11:16,328 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:16,328 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:11:16,328 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960676328"}]},"ts":"9223372036854775807"} 2024-11-18T20:11:16,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:11:16,330 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fc81ed96fcb7834182497904e66e6d77, NAME => 'TestAcidGuarantees,,1731960650867.fc81ed96fcb7834182497904e66e6d77.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:11:16,330 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:11:16,330 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960676330"}]},"ts":"9223372036854775807"} 2024-11-18T20:11:16,331 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:11:16,332 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:11:16,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-18T20:11:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:11:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:11:16,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:11:16,402 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 240), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=823 (was 819) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1606 (was 1878) 2024-11-18T20:11:16,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T20:11:16,402 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:11:16,402 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:11:16,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:16,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:16,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:11:16,403 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T20:11:16,403 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-11-18T20:11:16,403 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d8f95bb18b58,42935,1731960496743 2024-11-18T20:11:16,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T20:11:16,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T20:11:16,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:16,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:16,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:11:16,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:11:16,439 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T20:11:16,440 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:11:16,441 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:11:16,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:16,441 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd8f95bb18b58,42447,1731960497538' ***** 2024-11-18T20:11:16,441 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(959): stopping server d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d8f95bb18b58:42447. 2024-11-18T20:11:16,442 DEBUG [RS:0;d8f95bb18b58:42447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:11:16,442 DEBUG [RS:0;d8f95bb18b58:42447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:16,442 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T20:11:16,443 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T20:11:16,444 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T20:11:16,444 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T20:11:16,444 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T20:11:16,451 INFO [regionserver/d8f95bb18b58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T20:11:16,451 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T20:11:16,451 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T20:11:16,452 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:16,452 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T20:11:16,452 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T20:11:16,452 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T20:11:16,452 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T20:11:16,452 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T20:11:16,452 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-18T20:11:16,491 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/info/b4a2ee5bfe6d4ed5b20be36a139b9adb is 95, key is TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d./info:/1731960531337/DeleteFamily/seqid=0 2024-11-18T20:11:16,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742463_1639 (size=6514) 2024-11-18T20:11:16,652 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:16,786 INFO [regionserver/d8f95bb18b58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T20:11:16,786 INFO [regionserver/d8f95bb18b58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T20:11:16,852 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:16,902 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/info/b4a2ee5bfe6d4ed5b20be36a139b9adb 2024-11-18T20:11:16,923 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/ns/c13e6afe8172419cb04a8bd67461eda2 is 93, key is TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d./ns:/1731960531295/DeleteFamily/seqid=0 2024-11-18T20:11:16,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742464_1640 (size=5906) 2024-11-18T20:11:17,052 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:17,095 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:11:17,252 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:17,332 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/ns/c13e6afe8172419cb04a8bd67461eda2 2024-11-18T20:11:17,354 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/rep_barrier/c0becbbd19d94137a93724be31ab2150 is 102, key is TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d./rep_barrier:/1731960531295/DeleteFamily/seqid=0 2024-11-18T20:11:17,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742465_1641 (size=6025) 2024-11-18T20:11:17,453 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:17,653 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:17,765 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/rep_barrier/c0becbbd19d94137a93724be31ab2150 2024-11-18T20:11:17,786 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/table/3a9429a41ac243a4964f396d37b8b8cf is 96, key is TestAcidGuarantees,,1731960501398.b989f3762007617cfc5051d63359b50d./table:/1731960531295/DeleteFamily/seqid=0 2024-11-18T20:11:17,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742466_1642 (size=5936) 2024-11-18T20:11:17,853 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:18,053 DEBUG [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:11:18,201 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/table/3a9429a41ac243a4964f396d37b8b8cf 2024-11-18T20:11:18,205 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/info/b4a2ee5bfe6d4ed5b20be36a139b9adb as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/info/b4a2ee5bfe6d4ed5b20be36a139b9adb 2024-11-18T20:11:18,208 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/info/b4a2ee5bfe6d4ed5b20be36a139b9adb, entries=12, sequenceid=90, filesize=6.4 K 2024-11-18T20:11:18,209 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/ns/c13e6afe8172419cb04a8bd67461eda2 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/ns/c13e6afe8172419cb04a8bd67461eda2 2024-11-18T20:11:18,212 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/ns/c13e6afe8172419cb04a8bd67461eda2, entries=8, sequenceid=90, filesize=5.8 K 2024-11-18T20:11:18,212 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/rep_barrier/c0becbbd19d94137a93724be31ab2150 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/rep_barrier/c0becbbd19d94137a93724be31ab2150 2024-11-18T20:11:18,215 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/rep_barrier/c0becbbd19d94137a93724be31ab2150, entries=6, sequenceid=90, filesize=5.9 K 2024-11-18T20:11:18,216 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/.tmp/table/3a9429a41ac243a4964f396d37b8b8cf as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/table/3a9429a41ac243a4964f396d37b8b8cf 2024-11-18T20:11:18,218 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/table/3a9429a41ac243a4964f396d37b8b8cf, entries=7, sequenceid=90, filesize=5.8 K 2024-11-18T20:11:18,219 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1767ms, sequenceid=90, compaction requested=false 2024-11-18T20:11:18,222 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-18T20:11:18,227 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:11:18,227 INFO [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T20:11:18,227 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731960676451Running coprocessor pre-close hooks at 1731960676452 (+1 ms)Disabling compacts and flushes for region at 1731960676452Disabling writes for close at 1731960676452Obtaining lock to block concurrent updates at 1731960676452Preparing flush snapshotting stores in 1588230740 at 1731960676452Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731960676452Flushing stores of hbase:meta,,1.1588230740 at 1731960676455 (+3 ms)Flushing 1588230740/info: creating writer at 1731960676456 (+1 ms)Flushing 1588230740/info: appending metadata at 1731960676490 (+34 ms)Flushing 1588230740/info: closing flushed file at 1731960676490Flushing 1588230740/ns: creating writer at 1731960676905 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731960676922 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731960676922Flushing 1588230740/rep_barrier: creating writer at 1731960677335 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731960677353 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731960677353Flushing 1588230740/table: creating writer at 1731960677767 (+414 ms)Flushing 1588230740/table: appending metadata at 1731960677785 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731960677785Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ab5d85c: reopening flushed file at 1731960678205 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e732895: reopening flushed file at 1731960678208 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@172b3567: reopening flushed file at 1731960678212 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53113ae5: reopening flushed file at 1731960678215 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1767ms, sequenceid=90, compaction requested=false at 1731960678219 (+4 ms)Writing region close event to WAL at 1731960678220 (+1 ms)Running coprocessor post-close hooks at 1731960678227 (+7 ms)Closed at 1731960678227 2024-11-18T20:11:18,227 DEBUG [RS_CLOSE_META-regionserver/d8f95bb18b58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T20:11:18,254 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(976): stopping server d8f95bb18b58,42447,1731960497538; all regions closed. 2024-11-18T20:11:18,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741834_1010 (size=25344) 2024-11-18T20:11:18,263 DEBUG [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/oldWALs 2024-11-18T20:11:18,263 INFO [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d8f95bb18b58%2C42447%2C1731960497538.meta:.meta(num 1731960500873) 2024-11-18T20:11:18,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741832_1008 (size=14701013) 2024-11-18T20:11:18,266 DEBUG [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/oldWALs 2024-11-18T20:11:18,266 INFO [RS:0;d8f95bb18b58:42447 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d8f95bb18b58%2C42447%2C1731960497538:(num 1731960499941) 2024-11-18T20:11:18,266 DEBUG [RS:0;d8f95bb18b58:42447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:11:18,266 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T20:11:18,266 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T20:11:18,267 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.ChoreService(370): Chore service for: regionserver/d8f95bb18b58:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T20:11:18,267 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T20:11:18,267 INFO [regionserver/d8f95bb18b58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T20:11:18,267 INFO [RS:0;d8f95bb18b58:42447 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42447 2024-11-18T20:11:18,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T20:11:18,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d8f95bb18b58,42447,1731960497538 2024-11-18T20:11:18,304 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T20:11:18,312 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d8f95bb18b58,42447,1731960497538] 2024-11-18T20:11:18,320 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d8f95bb18b58,42447,1731960497538 already deleted, retry=false 2024-11-18T20:11:18,320 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d8f95bb18b58,42447,1731960497538 expired; onlineServers=0 2024-11-18T20:11:18,321 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd8f95bb18b58,42935,1731960496743' ***** 2024-11-18T20:11:18,321 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T20:11:18,321 INFO [M:0;d8f95bb18b58:42935 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T20:11:18,321 INFO [M:0;d8f95bb18b58:42935 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T20:11:18,321 DEBUG [M:0;d8f95bb18b58:42935 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T20:11:18,321 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T20:11:18,321 DEBUG [M:0;d8f95bb18b58:42935 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T20:11:18,321 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster-HFileCleaner.large.0-1731960499575 {}] cleaner.HFileCleaner(306): Exit Thread[master/d8f95bb18b58:0:becomeActiveMaster-HFileCleaner.large.0-1731960499575,5,FailOnTimeoutGroup] 2024-11-18T20:11:18,321 DEBUG [master/d8f95bb18b58:0:becomeActiveMaster-HFileCleaner.small.0-1731960499576 {}] cleaner.HFileCleaner(306): Exit Thread[master/d8f95bb18b58:0:becomeActiveMaster-HFileCleaner.small.0-1731960499576,5,FailOnTimeoutGroup] 2024-11-18T20:11:18,322 INFO [M:0;d8f95bb18b58:42935 {}] hbase.ChoreService(370): Chore service for: master/d8f95bb18b58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T20:11:18,322 INFO [M:0;d8f95bb18b58:42935 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T20:11:18,322 DEBUG [M:0;d8f95bb18b58:42935 {}] master.HMaster(1795): Stopping service threads 2024-11-18T20:11:18,322 INFO [M:0;d8f95bb18b58:42935 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T20:11:18,322 INFO [M:0;d8f95bb18b58:42935 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T20:11:18,322 ERROR [M:0;d8f95bb18b58:42935 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-18T20:11:18,322 INFO [M:0;d8f95bb18b58:42935 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T20:11:18,322 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T20:11:18,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T20:11:18,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:18,329 DEBUG [M:0;d8f95bb18b58:42935 {}] zookeeper.ZKUtil(347): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T20:11:18,329 WARN [M:0;d8f95bb18b58:42935 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T20:11:18,329 INFO [M:0;d8f95bb18b58:42935 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/.lastflushedseqids 2024-11-18T20:11:18,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742467_1643 (size=72) 2024-11-18T20:11:18,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:11:18,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42447-0x10150d991190001, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:11:18,413 INFO [RS:0;d8f95bb18b58:42447 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T20:11:18,413 INFO [RS:0;d8f95bb18b58:42447 {}] regionserver.HRegionServer(1031): Exiting; stopping=d8f95bb18b58,42447,1731960497538; zookeeper connection closed. 2024-11-18T20:11:18,413 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1191e6b6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1191e6b6 2024-11-18T20:11:18,414 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T20:11:18,738 INFO [M:0;d8f95bb18b58:42935 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T20:11:18,738 INFO [M:0;d8f95bb18b58:42935 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T20:11:18,739 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T20:11:18,739 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:18,739 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:18,739 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T20:11:18,739 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:18,739 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=757.45 KB heapSize=930.43 KB 2024-11-18T20:11:18,755 DEBUG [M:0;d8f95bb18b58:42935 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2ac4a7d2ca2a4b89a682bc75a5d9b9a4 is 82, key is hbase:meta,,1/info:regioninfo/1731960500964/Put/seqid=0 2024-11-18T20:11:18,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742468_1644 (size=5672) 2024-11-18T20:11:19,161 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2138 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2ac4a7d2ca2a4b89a682bc75a5d9b9a4 2024-11-18T20:11:19,185 DEBUG [M:0;d8f95bb18b58:42935 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed14b0f8c5e3468ab546930d0e9ad532 is 2276, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1731960534537/Put/seqid=0 2024-11-18T20:11:19,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742469_1645 (size=43514) 2024-11-18T20:11:19,188 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=756.89 KB at sequenceid=2138 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed14b0f8c5e3468ab546930d0e9ad532 2024-11-18T20:11:19,192 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed14b0f8c5e3468ab546930d0e9ad532 2024-11-18T20:11:19,215 DEBUG [M:0;d8f95bb18b58:42935 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/51c14d2d3d514159b9e7bb11619e58da is 69, key is d8f95bb18b58,42447,1731960497538/rs:state/1731960499701/Put/seqid=0 2024-11-18T20:11:19,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073742470_1646 (size=5156) 2024-11-18T20:11:19,625 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2138 (bloomFilter=true), to=hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/51c14d2d3d514159b9e7bb11619e58da 2024-11-18T20:11:19,629 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2ac4a7d2ca2a4b89a682bc75a5d9b9a4 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2ac4a7d2ca2a4b89a682bc75a5d9b9a4 2024-11-18T20:11:19,633 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2ac4a7d2ca2a4b89a682bc75a5d9b9a4, entries=8, sequenceid=2138, filesize=5.5 K 2024-11-18T20:11:19,634 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ed14b0f8c5e3468ab546930d0e9ad532 as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed14b0f8c5e3468ab546930d0e9ad532 2024-11-18T20:11:19,637 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed14b0f8c5e3468ab546930d0e9ad532 2024-11-18T20:11:19,637 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ed14b0f8c5e3468ab546930d0e9ad532, entries=166, sequenceid=2138, filesize=42.5 K 2024-11-18T20:11:19,638 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/51c14d2d3d514159b9e7bb11619e58da as hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/51c14d2d3d514159b9e7bb11619e58da 2024-11-18T20:11:19,642 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40539/user/jenkins/test-data/5777545a-cd02-1323-61fa-c644b9b2d86c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/51c14d2d3d514159b9e7bb11619e58da, entries=1, sequenceid=2138, filesize=5.0 K 2024-11-18T20:11:19,642 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(3140): Finished flush of dataSize ~757.45 KB/775626, heapSize ~930.13 KB/952456, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 903ms, sequenceid=2138, compaction requested=false 2024-11-18T20:11:19,644 INFO [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:19,644 DEBUG [M:0;d8f95bb18b58:42935 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731960678739Disabling compacts and flushes for region at 1731960678739Disabling writes for close at 1731960678739Obtaining lock to block concurrent updates at 1731960678739Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731960678739Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=775626, getHeapSize=952696, getOffHeapSize=0, getCellsCount=2329 at 1731960678739Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731960678740 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731960678740Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731960678755 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731960678755Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731960679164 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731960679184 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731960679184Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731960679192 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731960679214 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731960679214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@409dc288: reopening flushed file at 1731960679628 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d0acce1: reopening flushed file at 1731960679633 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4131405e: reopening flushed file at 1731960679638 (+5 ms)Finished flush of dataSize ~757.45 KB/775626, heapSize ~930.13 KB/952456, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 903ms, sequenceid=2138, compaction requested=false at 1731960679642 (+4 ms)Writing region close event to WAL at 1731960679643 (+1 ms)Closed at 1731960679643 2024-11-18T20:11:19,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35355 is added to blk_1073741830_1006 (size=915779) 2024-11-18T20:11:19,650 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T20:11:19,650 INFO [M:0;d8f95bb18b58:42935 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T20:11:19,650 INFO [M:0;d8f95bb18b58:42935 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42935 2024-11-18T20:11:19,651 INFO [M:0;d8f95bb18b58:42935 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T20:11:19,787 INFO [M:0;d8f95bb18b58:42935 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T20:11:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:11:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10150d991190000, quorum=127.0.0.1:58690, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:11:19,793 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T20:11:19,795 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T20:11:19,795 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T20:11:19,795 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T20:11:19,796 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/hadoop.log.dir/,STOPPED} 2024-11-18T20:11:19,800 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T20:11:19,800 WARN [BP-891117525-172.17.0.2-1731960493244 heartbeating to localhost/127.0.0.1:40539 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T20:11:19,800 WARN [BP-891117525-172.17.0.2-1731960493244 heartbeating to localhost/127.0.0.1:40539 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-891117525-172.17.0.2-1731960493244 (Datanode Uuid 1977cf0f-cc6f-41c3-863b-3bc2700219f3) service to localhost/127.0.0.1:40539 2024-11-18T20:11:19,800 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T20:11:19,803 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/data/data1/current/BP-891117525-172.17.0.2-1731960493244 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T20:11:19,804 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/cluster_ae13d1dd-650f-0cac-ae3c-a0cbbaf99295/data/data2/current/BP-891117525-172.17.0.2-1731960493244 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T20:11:19,804 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T20:11:19,813 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T20:11:19,813 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T20:11:19,813 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T20:11:19,814 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T20:11:19,814 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565bf4ca-7ad9-444a-ef9f-cb1e64cc314c/hadoop.log.dir/,STOPPED} 2024-11-18T20:11:19,833 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T20:11:20,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down